Rollup merge of #134232 - bjorn3:naked_asm_improvements, r=wesleywiser

Share the naked asm impl between cg_ssa and cg_clif

This was introduced in https://github.com/rust-lang/rust/pull/128004.

compiler/rustc_codegen_ssa/src/base.rs

@@ -12,19 +12,21 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
 use rustc_data_structures::sync::{IntoDynSyncSend, par_map};
 use rustc_data_structures::unord::UnordMap;
+use rustc_hir::ItemId;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::lang_items::LangItem;
 use rustc_metadata::EncodedMetadata;
-use rustc_middle::bug;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, DebuggerVisualizerType};
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
 use rustc_middle::middle::{exported_symbols, lang_items};
 use rustc_middle::mir::BinOp;
+use rustc_middle::mir::interpret::ErrorHandled;
 use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem, MonoItemPartitions};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug};
 use rustc_session::Session;
 use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
 use rustc_span::{DUMMY_SP, Symbol, sym};
@@ -417,6 +419,69 @@ pub(crate) fn codegen_instance<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>(
     mir::codegen_mir::<Bx>(cx, instance);
 }
 
+pub fn codegen_global_asm<'tcx, Cx>(cx: &mut Cx, item_id: ItemId)
+where
+    Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> + AsmCodegenMethods<'tcx>,
+{
+    let item = cx.tcx().hir_item(item_id);
+    if let rustc_hir::ItemKind::GlobalAsm { asm, .. } = item.kind {
+        let operands: Vec<_> = asm
+            .operands
+            .iter()
+            .map(|(op, op_sp)| match *op {
+                rustc_hir::InlineAsmOperand::Const { ref anon_const } => {
+                    match cx.tcx().const_eval_poly(anon_const.def_id.to_def_id()) {
+                        Ok(const_value) => {
+                            let ty =
+                                cx.tcx().typeck_body(anon_const.body).node_type(anon_const.hir_id);
+                            let string = common::asm_const_to_str(
+                                cx.tcx(),
+                                *op_sp,
+                                const_value,
+                                cx.layout_of(ty),
+                            );
+                            GlobalAsmOperandRef::Const { string }
+                        }
+                        Err(ErrorHandled::Reported { .. }) => {
+                            // An error has already been reported and
+                            // compilation is guaranteed to fail if execution
+                            // hits this path. So an empty string instead of
+                            // a stringified constant value will suffice.
+                            GlobalAsmOperandRef::Const { string: String::new() }
+                        }
+                        Err(ErrorHandled::TooGeneric(_)) => {
+                            span_bug!(*op_sp, "asm const cannot be resolved; too generic")
+                        }
+                    }
+                }
+                rustc_hir::InlineAsmOperand::SymFn { expr } => {
+                    let ty = cx.tcx().typeck(item_id.owner_id).expr_ty(expr);
+                    let instance = match ty.kind() {
+                        &ty::FnDef(def_id, args) => Instance::new(def_id, args),
+                        _ => span_bug!(*op_sp, "asm sym is not a function"),
+                    };
+
+                    GlobalAsmOperandRef::SymFn { instance }
+                }
+                rustc_hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
+                    GlobalAsmOperandRef::SymStatic { def_id }
+                }
+                rustc_hir::InlineAsmOperand::In { .. }
+                | rustc_hir::InlineAsmOperand::Out { .. }
+                | rustc_hir::InlineAsmOperand::InOut { .. }
+                | rustc_hir::InlineAsmOperand::SplitInOut { .. }
+                | rustc_hir::InlineAsmOperand::Label { .. } => {
+                    span_bug!(*op_sp, "invalid operand type for global_asm!")
+                }
+            })
+            .collect();
+
+        cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
+    } else {
+        span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
+    }
+}
+
 /// Creates the `main` function which will initialize the rust runtime and call
 /// users main function.
 pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(

compiler/rustc_codegen_ssa/src/mir/mod.rs

@@ -20,7 +20,7 @@ mod coverageinfo;
 pub mod debuginfo;
 mod intrinsic;
 mod locals;
-mod naked_asm;
+pub mod naked_asm;
 pub mod operand;
 pub mod place;
 mod rvalue;
@@ -178,11 +178,6 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
     debug!("fn_abi: {:?}", fn_abi);
 
-    if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) {
-        crate::mir::naked_asm::codegen_naked_asm::<Bx>(cx, &mir, instance);
-        return;
-    }
-
     if tcx.features().ergonomic_clones() {
         let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions(
             tcx,

compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

@@ -1,23 +1,33 @@
 use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind};
 use rustc_attr_parsing::InstructionSetAttr;
 use rustc_hir::def_id::DefId;
-use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility};
-use rustc_middle::mir::{Body, InlineAsmOperand};
-use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
-use rustc_middle::ty::{Instance, Ty, TyCtxt};
+use rustc_middle::mir::mono::{Linkage, MonoItemData, Visibility};
+use rustc_middle::mir::{InlineAsmOperand, START_BLOCK};
+use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
+use rustc_middle::ty::{Instance, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::{bug, span_bug, ty};
 use rustc_span::sym;
 use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
 use rustc_target::spec::{BinaryFormat, WasmCAbi};
 
 use crate::common;
-use crate::traits::{AsmCodegenMethods, BuilderMethods, GlobalAsmOperandRef, MiscCodegenMethods};
+use crate::mir::AsmCodegenMethods;
+use crate::traits::GlobalAsmOperandRef;
 
-pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    cx: &'a Bx::CodegenCx,
-    mir: &Body<'tcx>,
+pub fn codegen_naked_asm<
+    'a,
+    'tcx,
+    Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
+        + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
+        + AsmCodegenMethods<'tcx>,
+>(
+    cx: &'a mut Cx,
     instance: Instance<'tcx>,
+    item_data: MonoItemData,
 ) {
+    assert!(!instance.args.has_infer());
+    let mir = cx.tcx().instance_mir(instance.def);
+
     let rustc_middle::mir::TerminatorKind::InlineAsm {
         asm_macro: _,
         template,
@@ -26,15 +36,14 @@ pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         line_spans,
         targets: _,
         unwind: _,
-    } = mir.basic_blocks.iter().next().unwrap().terminator().kind
+    } = mir.basic_blocks[START_BLOCK].terminator().kind
     else {
         bug!("#[naked] functions should always terminate with an asm! block")
     };
 
     let operands: Vec<_> =
-        operands.iter().map(|op| inline_to_global_operand::<Bx>(cx, instance, op)).collect();
+        operands.iter().map(|op| inline_to_global_operand::<Cx>(cx, instance, op)).collect();
 
-    let item_data = cx.codegen_unit().items().get(&MonoItem::Fn(instance)).unwrap();
     let name = cx.mangled_name(instance);
     let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
     let (begin, end) = prefix_and_suffix(cx.tcx(), instance, &name, item_data, fn_abi);
@@ -47,8 +56,8 @@ pub(crate) fn codegen_naked_asm<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     cx.codegen_global_asm(&template_vec, &operands, options, line_spans);
 }
 
-fn inline_to_global_operand<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
-    cx: &'a Bx::CodegenCx,
+fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>(
+    cx: &'a Cx,
     instance: Instance<'tcx>,
     op: &InlineAsmOperand<'tcx>,
 ) -> GlobalAsmOperandRef<'tcx> {
@@ -108,7 +117,7 @@ fn prefix_and_suffix<'tcx>(
     tcx: TyCtxt<'tcx>,
     instance: Instance<'tcx>,
     asm_name: &str,
-    item_data: &MonoItemData,
+    item_data: MonoItemData,
     fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
 ) -> (String, String) {
     use std::fmt::Write;
@@ -210,8 +219,10 @@ fn prefix_and_suffix<'tcx>(
             writeln!(begin, ".pushsection {section},\"ax\", {progbits}").unwrap();
             writeln!(begin, ".balign {align_bytes}").unwrap();
             write_linkage(&mut begin).unwrap();
-            if let Visibility::Hidden = item_data.visibility {
-                writeln!(begin, ".hidden {asm_name}").unwrap();
+            match item_data.visibility {
+                Visibility::Default => {}
+                Visibility::Protected => writeln!(begin, ".protected {asm_name}").unwrap(),
+                Visibility::Hidden => writeln!(begin, ".hidden {asm_name}").unwrap(),
             }
             writeln!(begin, ".type {asm_name}, {function}").unwrap();
             if !arch_prefix.is_empty() {
@@ -231,8 +242,9 @@ fn prefix_and_suffix<'tcx>(
             writeln!(begin, ".pushsection {},regular,pure_instructions", section).unwrap();
             writeln!(begin, ".balign {align_bytes}").unwrap();
             write_linkage(&mut begin).unwrap();
-            if let Visibility::Hidden = item_data.visibility {
-                writeln!(begin, ".private_extern {asm_name}").unwrap();
+            match item_data.visibility {
+                Visibility::Default | Visibility::Protected => {}
+                Visibility::Hidden => writeln!(begin, ".private_extern {asm_name}").unwrap(),
             }
             writeln!(begin, "{asm_name}:").unwrap();
 

compiler/rustc_codegen_ssa/src/mono_item.rs

@@ -1,17 +1,18 @@
-use rustc_hir as hir;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
-use rustc_middle::mir::interpret::ErrorHandled;
-use rustc_middle::mir::mono::{Linkage, MonoItem, Visibility};
-use rustc_middle::ty::Instance;
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
-use rustc_middle::{span_bug, ty};
+use rustc_middle::mir::mono::{Linkage, MonoItem, MonoItemData, Visibility};
+use rustc_middle::ty::layout::HasTyCtxt;
 use tracing::debug;
 
+use crate::base;
+use crate::mir::naked_asm;
 use crate::traits::*;
-use crate::{base, common};
 
 pub trait MonoItemExt<'a, 'tcx> {
-    fn define<Bx: BuilderMethods<'a, 'tcx>>(&self, cx: &'a Bx::CodegenCx);
+    fn define<Bx: BuilderMethods<'a, 'tcx>>(
+        &self,
+        cx: &'a mut Bx::CodegenCx,
+        item_data: MonoItemData,
+    );
     fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         cx: &'a Bx::CodegenCx,
@@ -22,7 +23,11 @@ pub trait MonoItemExt<'a, 'tcx> {
 }
 
 impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
-    fn define<Bx: BuilderMethods<'a, 'tcx>>(&self, cx: &'a Bx::CodegenCx) {
+    fn define<Bx: BuilderMethods<'a, 'tcx>>(
+        &self,
+        cx: &'a mut Bx::CodegenCx,
+        item_data: MonoItemData,
+    ) {
         debug!(
             "BEGIN IMPLEMENTING '{} ({})' in cgu {}",
             self,
@@ -35,71 +40,19 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
                 cx.codegen_static(def_id);
             }
             MonoItem::GlobalAsm(item_id) => {
-                let item = cx.tcx().hir_item(item_id);
-                if let hir::ItemKind::GlobalAsm { asm, .. } = item.kind {
-                    let operands: Vec<_> = asm
-                        .operands
-                        .iter()
-                        .map(|(op, op_sp)| match *op {
-                            hir::InlineAsmOperand::Const { ref anon_const } => {
-                                match cx.tcx().const_eval_poly(anon_const.def_id.to_def_id()) {
-                                    Ok(const_value) => {
-                                        let ty = cx
-                                            .tcx()
-                                            .typeck_body(anon_const.body)
-                                            .node_type(anon_const.hir_id);
-                                        let string = common::asm_const_to_str(
-                                            cx.tcx(),
-                                            *op_sp,
-                                            const_value,
-                                            cx.layout_of(ty),
-                                        );
-                                        GlobalAsmOperandRef::Const { string }
-                                    }
-                                    Err(ErrorHandled::Reported { .. }) => {
-                                        // An error has already been reported and
-                                        // compilation is guaranteed to fail if execution
-                                        // hits this path. So an empty string instead of
-                                        // a stringified constant value will suffice.
-                                        GlobalAsmOperandRef::Const { string: String::new() }
-                                    }
-                                    Err(ErrorHandled::TooGeneric(_)) => {
-                                        span_bug!(
-                                            *op_sp,
-                                            "asm const cannot be resolved; too generic"
-                                        )
-                                    }
-                                }
-                            }
-                            hir::InlineAsmOperand::SymFn { expr } => {
-                                let ty = cx.tcx().typeck(item_id.owner_id).expr_ty(expr);
-                                let instance = match ty.kind() {
-                                    &ty::FnDef(def_id, args) => Instance::new(def_id, args),
-                                    _ => span_bug!(*op_sp, "asm sym is not a function"),
-                                };
-
-                                GlobalAsmOperandRef::SymFn { instance }
-                            }
-                            hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
-                                GlobalAsmOperandRef::SymStatic { def_id }
-                            }
-                            hir::InlineAsmOperand::In { .. }
-                            | hir::InlineAsmOperand::Out { .. }
-                            | hir::InlineAsmOperand::InOut { .. }
-                            | hir::InlineAsmOperand::SplitInOut { .. }
-                            | hir::InlineAsmOperand::Label { .. } => {
-                                span_bug!(*op_sp, "invalid operand type for global_asm!")
-                            }
-                        })
-                        .collect();
-
-                    cx.codegen_global_asm(asm.template, &operands, asm.options, asm.line_spans);
-                } else {
-                    span_bug!(item.span, "Mismatch between hir::Item type and MonoItem type")
-                }
+                base::codegen_global_asm(cx, item_id);
             }
             MonoItem::Fn(instance) => {
-                base::codegen_instance::<Bx>(cx, instance);
+                if cx
+                    .tcx()
+                    .codegen_fn_attrs(instance.def_id())
+                    .flags
+                    .contains(CodegenFnAttrFlags::NAKED)
+                {
+                    naked_asm::codegen_naked_asm::<Bx::CodegenCx>(cx, instance, item_data);
+                } else {
+                    base::codegen_instance::<Bx>(cx, instance);
+                }
             }
         }
 

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -62,7 +62,7 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
 
 pub trait AsmCodegenMethods<'tcx> {
     fn codegen_global_asm(
-        &self,
+        &mut self,
         template: &[InlineAsmTemplatePiece],
         operands: &[GlobalAsmOperandRef<'tcx>],
         options: InlineAsmOptions,