Rollup of 4 pull requests

compiler/rustc_abi/src/lib.rs

@@ -1753,6 +1753,9 @@ pub struct AddressSpace(pub u32);
 impl AddressSpace {
     /// LLVM's `0` address space.
     pub const ZERO: Self = AddressSpace(0);
+    /// The address space for workgroup memory on nvptx and amdgpu.
+    /// See e.g. the `gpu_launch_sized_workgroup_mem` intrinsic for details.
+    pub const GPU_WORKGROUP: Self = AddressSpace(3);
 }
 
 /// How many scalable vectors are in a `BackendRepr::ScalableVector`?

compiler/rustc_ast_lowering/src/expr.rs

@@ -1052,7 +1052,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         binder: &ClosureBinder,
         capture_clause: CaptureBy,
         closure_id: NodeId,
-        mut constness: Const,
+        constness: Const,
         movability: Movability,
         decl: &FnDecl,
         body: &Expr,
@@ -1062,18 +1062,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let closure_def_id = self.local_def_id(closure_id);
         let (binder_clause, generic_params) = self.lower_closure_binder(binder);
 
-        if let Const::Yes(span) = constness {
-            if !self.is_in_const_context {
-                self.dcx().span_err(span, "cannot use `const` closures outside of const contexts");
-                constness = Const::No;
-            }
-        }
-
         let (body_id, closure_kind) = self.with_new_scopes(fn_decl_span, move |this| {
             let mut coroutine_kind = find_attr!(attrs, Coroutine(_) => hir::CoroutineKind::Coroutine(Movability::Movable));
 
             // FIXME(contracts): Support contracts on closures?
-            let body_id = this.lower_fn_body(decl, None, constness, |this| {
+            let body_id = this.lower_fn_body(decl, None, |this| {
                 this.coroutine_kind = coroutine_kind;
                 let e = this.lower_expr_mut(body);
                 coroutine_kind = this.coroutine_kind;

compiler/rustc_ast_lowering/src/item.rs

@@ -1,4 +1,3 @@
-use std::mem;
 
 use rustc_abi::ExternAbi;
 use rustc_ast::visit::AssocCtxt;
@@ -378,7 +377,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         body.as_deref(),
                         attrs,
                         contract.as_deref(),
-                        header.constness,
                     );
 
                     let itctx = ImplTraitContext::Universal;
@@ -1068,7 +1066,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     Some(body),
                     attrs,
                     contract.as_deref(),
-                    sig.header.constness,
                 );
                 let (generics, sig) = self.lower_method_sig(
                     generics,
@@ -1262,7 +1259,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     body.as_deref(),
                     attrs,
                     contract.as_deref(),
-                    sig.header.constness,
                 );
                 let (generics, sig) = self.lower_method_sig(
                     generics,
@@ -1391,13 +1387,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         f: impl FnOnce(&mut Self) -> (&'hir [hir::Param<'hir>], hir::Expr<'hir>),
     ) -> hir::BodyId {
         let prev_coroutine_kind = self.coroutine_kind.take();
-        let prev_is_in_const_context = mem::take(&mut self.is_in_const_context);
         let task_context = self.task_context.take();
         let (parameters, result) = f(self);
         let body_id = self.record_body(parameters, result);
         self.task_context = task_context;
         self.coroutine_kind = prev_coroutine_kind;
-        self.is_in_const_context = prev_is_in_const_context;
         body_id
     }
 
@@ -1416,13 +1410,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
         &mut self,
         decl: &FnDecl,
         contract: Option<&FnContract>,
-        constness: Const,
         body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
     ) -> hir::BodyId {
         self.lower_body(|this| {
-            if let Const::Yes(_) = constness {
-                this.is_in_const_context = true;
-            }
             let params =
                 this.arena.alloc_from_iter(decl.inputs.iter().map(|x| this.lower_param(x)));
 
@@ -1440,20 +1430,16 @@ impl<'hir> LoweringContext<'_, 'hir> {
         decl: &FnDecl,
         body: &Block,
         contract: Option<&FnContract>,
-        constness: Const,
     ) -> hir::BodyId {
-        self.lower_fn_body(decl, contract, constness, |this| this.lower_block_expr(body))
+        self.lower_fn_body(decl, contract, |this| this.lower_block_expr(body))
     }
 
     pub(super) fn lower_const_body(&mut self, span: Span, expr: Option<&Expr>) -> hir::BodyId {
         self.lower_body(|this| {
             (
                 &[],
                 match expr {
-                    Some(expr) => {
-                        this.is_in_const_context = true;
-                        this.lower_expr_mut(expr)
-                    }
+                    Some(expr) => this.lower_expr_mut(expr),
                     None => this.expr_err(span, this.dcx().span_delayed_bug(span, "no block")),
                 },
             )
@@ -1472,13 +1458,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         body: Option<&Block>,
         attrs: &'hir [hir::Attribute],
         contract: Option<&FnContract>,
-        constness: Const,
     ) -> hir::BodyId {
         let Some(body) = body else {
             // Functions without a body are an error, except if this is an intrinsic. For those we
             // create a fake body so that the entire rest of the compiler doesn't have to deal with
             // this as a special case.
-            return self.lower_fn_body(decl, contract, constness, |this| {
+            return self.lower_fn_body(decl, contract, |this| {
                 if find_attr!(attrs, RustcIntrinsic) || this.tcx.is_sdylib_interface_build() {
                     let span = this.lower_span(span);
                     let empty_block = hir::Block {
@@ -1503,7 +1488,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         };
         let Some(coroutine_kind) = coroutine_kind else {
             // Typical case: not a coroutine.
-            return self.lower_fn_body_block(decl, body, contract, constness);
+            return self.lower_fn_body_block(decl, body, contract);
         };
         // FIXME(contracts): Support contracts on async fn.
         self.lower_body(|this| {

compiler/rustc_ast_lowering/src/lib.rs

@@ -124,7 +124,6 @@ struct LoweringContext<'a, 'hir> {
     loop_scope: Option<HirId>,
     is_in_loop_condition: bool,
     is_in_dyn_type: bool,
-    is_in_const_context: bool,
 
     current_hir_id_owner: hir::OwnerId,
     item_local_id_counter: hir::ItemLocalId,
@@ -193,7 +192,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             loop_scope: None,
             is_in_loop_condition: false,
             is_in_dyn_type: false,
-            is_in_const_context: false,
             coroutine_kind: None,
             task_context: None,
             current_item: None,

compiler/rustc_codegen_llvm/src/declare.rs

@@ -14,6 +14,7 @@
 use std::borrow::Borrow;
 
 use itertools::Itertools;
+use rustc_abi::AddressSpace;
 use rustc_codegen_ssa::traits::{MiscCodegenMethods, TypeMembershipCodegenMethods};
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::ty::{Instance, Ty};
@@ -104,6 +105,28 @@ impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
             )
         }
     }
+
+    /// Declare a global value in a specific address space.
+    ///
+    /// If there’s a value with the same name already declared, the function will
+    /// return its Value instead.
+    pub(crate) fn declare_global_in_addrspace(
+        &self,
+        name: &str,
+        ty: &'ll Type,
+        addr_space: AddressSpace,
+    ) -> &'ll Value {
+        debug!("declare_global(name={name:?}, addrspace={addr_space:?})");
+        unsafe {
+            llvm::LLVMRustGetOrInsertGlobalInAddrspace(
+                (**self).borrow().llmod,
+                name.as_c_char_ptr(),
+                name.len(),
+                ty,
+                addr_space.0,
+            )
+        }
+    }
 }
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -3,8 +3,8 @@ use std::ffi::c_uint;
 use std::{assert_matches, iter, ptr};
 
 use rustc_abi::{
-    Align, BackendRepr, Float, HasDataLayout, Integer, NumScalableVectors, Primitive, Size,
-    WrappingRange,
+    AddressSpace, Align, BackendRepr, Float, HasDataLayout, Integer, NumScalableVectors, Primitive,
+    Size, WrappingRange,
 };
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
@@ -178,6 +178,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         span: Span,
     ) -> Result<(), ty::Instance<'tcx>> {
         let tcx = self.tcx;
+        let llvm_version = crate::llvm_util::get_version();
 
         let name = tcx.item_name(instance.def_id());
         let fn_args = instance.args;
@@ -194,7 +195,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::maximum_number_nsz_f64
             | sym::maximum_number_nsz_f128
                 // Need at least LLVM 22 for `min/maximumnum` to not crash LLVM.
-                if crate::llvm_util::get_version() >= (22, 0, 0) =>
+                if llvm_version >= (22, 0, 0) =>
             {
                 let intrinsic_name = if name.as_str().starts_with("min") {
                     "llvm.minimumnum"
@@ -420,7 +421,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             }
 
             // FIXME move into the branch below when LLVM 22 is the lowest version we support.
-            sym::carryless_mul if crate::llvm_util::get_version() >= (22, 0, 0) => {
+            sym::carryless_mul if llvm_version >= (22, 0, 0) => {
                 let ty = args[0].layout.ty;
                 if !ty.is_integral() {
                     tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
@@ -620,6 +621,46 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 return Ok(());
             }
 
+            sym::gpu_launch_sized_workgroup_mem => {
+                // Generate an anonymous global per call, with these properties:
+                // 1. The global is in the address space for workgroup memory
+                // 2. It is an `external` global
+                // 3. It is correctly aligned for the pointee `T`
+                // All instances of extern addrspace(gpu_workgroup) globals are merged in the LLVM backend.
+                // The name is irrelevant.
+                // See https://docs.nvidia.com/cuda/cuda-c-programming-guide/#shared
+                let name = if llvm_version < (23, 0, 0) && tcx.sess.target.arch == Arch::Nvptx64 {
+                    // The auto-assigned name for extern shared globals in the nvptx backend does
+                    // not compile in ptxas. Workaround this issue by assigning a name.
+                    // Fixed in LLVM 23.
+                    "gpu_launch_sized_workgroup_mem"
+                } else {
+                    ""
+                };
+                let global = self.declare_global_in_addrspace(
+                    name,
+                    self.type_array(self.type_i8(), 0),
+                    AddressSpace::GPU_WORKGROUP,
+                );
+                let ty::RawPtr(inner_ty, _) = result.layout.ty.kind() else { unreachable!() };
+                // The alignment of the global is used to specify the *minimum* alignment that
+                // must be obeyed by the GPU runtime.
+                // When multiple of these global variables are used by a kernel, the maximum alignment is taken.
+                // See https://github.com/llvm/llvm-project/blob/a271d07488a85ce677674bbe8101b10efff58c95/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp#L821
+                let alignment = self.align_of(*inner_ty).bytes() as u32;
+                unsafe {
+                    // FIXME Workaround the above issue by taking maximum alignment if the global existed
+                    if tcx.sess.target.arch == Arch::Nvptx64 {
+                        if alignment > llvm::LLVMGetAlignment(global) {
+                            llvm::LLVMSetAlignment(global, alignment);
+                        }
+                    } else {
+                        llvm::LLVMSetAlignment(global, alignment);
+                    }
+                }
+                self.cx().const_pointercast(global, self.type_ptr())
+            }
+
             sym::amdgpu_dispatch_ptr => {
                 let val = self.call_intrinsic("llvm.amdgcn.dispatch.ptr", &[], &[]);
                 // Relying on `LLVMBuildPointerCast` to produce an addrspacecast

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2003,6 +2003,13 @@ unsafe extern "C" {
         NameLen: size_t,
         T: &'a Type,
     ) -> &'a Value;
+    pub(crate) fn LLVMRustGetOrInsertGlobalInAddrspace<'a>(
+        M: &'a Module,
+        Name: *const c_char,
+        NameLen: size_t,
+        T: &'a Type,
+        AddressSpace: c_uint,
+    ) -> &'a Value;
     pub(crate) fn LLVMRustGetNamedValue(
         M: &Module,
         Name: *const c_char,

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -111,6 +111,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 sym::abort
                 | sym::unreachable
                 | sym::cold_path
+                | sym::gpu_launch_sized_workgroup_mem
                 | sym::breakpoint
                 | sym::amdgpu_dispatch_ptr
                 | sym::assert_zero_valid

compiler/rustc_const_eval/src/const_eval/fn_queries.rs

@@ -16,7 +16,13 @@ fn constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Constness {
             // Foreign functions cannot be evaluated at compile-time.
             Constness::NotConst
         }
-        Node::Expr(e) if let ExprKind::Closure(c) = e.kind => c.constness,
+        Node::Expr(e) if let ExprKind::Closure(c) = e.kind => {
+            if let Constness::Const = c.constness && tcx.hir_body_const_context(tcx.local_parent(def_id)).is_none() {
+                tcx.dcx().span_err(tcx.def_span(def_id), "cannot use `const` closures outside of const contexts");
+                return Constness::NotConst;
+            }
+            c.constness
+        },
         // FIXME(fee1-dead): extract this one out and rename this query to `fn_constness` so we don't need `is_const_fn` anymore.
         Node::Item(i) if let ItemKind::Impl(impl_) = i.kind => impl_.constness,
         Node::Item(Item { kind: ItemKind::Fn { sig, .. }, .. }) => sig.header.constness,

compiler/rustc_hir_analysis/src/check/always_applicable.rs

@@ -11,7 +11,7 @@ use rustc_infer::infer::{RegionResolutionError, TyCtxtInferExt};
 use rustc_infer::traits::{ObligationCause, ObligationCauseCode};
 use rustc_middle::span_bug;
 use rustc_middle::ty::util::CheckRegions;
-use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypingMode};
+use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeVisitableExt, TypingMode};
 use rustc_trait_selection::regions::InferCtxtRegionExt;
 use rustc_trait_selection::traits::{self, ObligationCtxt};
 
@@ -65,6 +65,8 @@ pub(crate) fn check_drop_impl(
                 adt_to_impl_args,
             )?;
 
+            ensure_all_fields_are_const_destruct(tcx, drop_impl_did, adt_def.did())?;
+
             ensure_impl_predicates_are_implied_by_item_defn(
                 tcx,
                 drop_impl_did,
@@ -173,6 +175,64 @@ fn ensure_impl_params_and_item_params_correspond<'tcx>(
     Err(err.emit())
 }
 
+fn ensure_all_fields_are_const_destruct<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    impl_def_id: LocalDefId,
+    adt_def_id: DefId,
+) -> Result<(), ErrorGuaranteed> {
+    if !tcx.is_conditionally_const(impl_def_id) {
+        return Ok(());
+    }
+    let infcx = tcx.infer_ctxt().build(TypingMode::non_body_analysis());
+    let ocx = ObligationCtxt::new_with_diagnostics(&infcx);
+
+    let impl_span = tcx.def_span(impl_def_id.to_def_id());
+    let env =
+        ty::EarlyBinder::bind(tcx.param_env(impl_def_id)).instantiate_identity().skip_norm_wip();
+    let args = ty::GenericArgs::identity_for_item(tcx, impl_def_id);
+    let destruct_trait = tcx.lang_items().destruct_trait().unwrap();
+    for field in tcx.adt_def(adt_def_id).all_fields() {
+        let field_ty = field.ty(tcx, args);
+        let cause = traits::ObligationCause::new(
+            tcx.def_span(field.did),
+            impl_def_id,
+            ObligationCauseCode::Misc,
+        );
+        ocx.register_obligation(traits::Obligation::new(
+            tcx,
+            cause,
+            env,
+            ty::ClauseKind::HostEffect(ty::HostEffectPredicate {
+                trait_ref: ty::TraitRef::new(tcx, destruct_trait, [field_ty]),
+                constness: ty::BoundConstness::Maybe,
+            }),
+        ));
+    }
+    ocx.evaluate_obligations_error_on_ambiguity()
+        .into_iter()
+        .map(|error| {
+            let ty::ClauseKind::HostEffect(eff) =
+                error.root_obligation.predicate.expect_clause().kind().no_bound_vars().unwrap()
+            else {
+                unreachable!()
+            };
+            let field_ty = eff.trait_ref.self_ty();
+            let diag = struct_span_code_err!(
+                tcx.dcx(),
+                error.root_obligation.cause.span,
+                E0367,
+                "`{field_ty}` does not implement `[const] Destruct`",
+            )
+            .with_span_note(impl_span, "required for this `Drop` impl");
+            if field_ty.has_param() {
+                // FIXME: suggest adding `[const] Destruct` by teaching
+                // `suggest_restricting_param_bound` about const traits.
+            }
+            Err(diag.emit())
+        })
+        .collect()
+}
+
 /// Confirms that all predicates defined on the `Drop` impl (`drop_impl_def_id`) are able to be
 /// proven from within `adt_def_id`'s environment. I.e. all the predicates on the impl are
 /// implied by the ADT being well formed.