rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

intrinsics: use const generic to set atomic ordering

Browse Source
This commit is contained in:
Ralf Jung
2025-05-28 18:37:58 +02:00
parent 5e0bdaa9dd
commit 8808c9d34b
23 changed files with 653 additions and 1474 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
compiler/rustc_codegen_ssa/src/errors.rs
View File

@@ -796,22 +796,6 @@ pub(crate) struct ShuffleIndicesEvaluation {
pub span: Span,
}
#[derive(Diagnostic)]
#[diag(codegen_ssa_missing_memory_ordering)]
pub(crate) struct MissingMemoryOrdering;
#[derive(Diagnostic)]
#[diag(codegen_ssa_unknown_atomic_ordering)]
pub(crate) struct UnknownAtomicOrdering;
#[derive(Diagnostic)]
#[diag(codegen_ssa_atomic_compare_exchange)]
pub(crate) struct AtomicCompareExchange;
#[derive(Diagnostic)]
#[diag(codegen_ssa_unknown_atomic_operation)]
pub(crate) struct UnknownAtomicOperation;
#[derive(Diagnostic)]
pub enum InvalidMonomorphization<'tcx> {
#[diag(codegen_ssa_invalid_monomorphization_basic_integer_type, code = E0511)]

338
compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
View File

@@ -8,9 +8,10 @@ use rustc_span::sym;
use super::FunctionCx;
use super::operand::OperandRef;
use super::place::PlaceRef;
use crate::common::{AtomicRmwBinOp, SynchronizationScope};
use crate::errors::InvalidMonomorphization;
use crate::traits::*;
use crate::{MemFlags, errors, meth, size_of_val};
use crate::{MemFlags, meth, size_of_val};
fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
bx: &mut Bx,
@@ -62,7 +63,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let span = source_info.span;
let name = bx.tcx().item_name(instance.def_id());
let name_str = name.as_str();
let fn_args = instance.args;
// If we're swapping something that's *not* an `OperandValue::Ref`,
@@ -89,14 +89,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}
let ret_llval = |bx: &mut Bx, llval| {
if result.layout.ty.is_bool() {
let val = bx.from_immediate(llval);
bx.store_to_place(val, result.val);
} else if !result.layout.ty.is_unit() {
bx.store_to_place(llval, result.val);
}
Ok(())
let invalid_monomorphization_int_type = |ty| {
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
};
let parse_atomic_ordering = |ord: ty::Value<'tcx>| {
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
discr.to_atomic_ordering()
};
let llval = match name {
@@ -336,183 +335,144 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
}
// This requires that atomic intrinsics follow a specific naming pattern:
// "atomic_<operation>[_<ordering>]"
name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
use rustc_middle::ty::AtomicOrdering::*;
use crate::common::{AtomicRmwBinOp, SynchronizationScope};
let invalid_monomorphization = |ty| {
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
span,
name,
ty,
});
};
let parse_const_generic_ordering = |ord: ty::Value<'tcx>| {
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
discr.to_atomic_ordering()
};
// Some intrinsics have the ordering already converted to a const generic parameter, we handle those first.
match name {
sym::atomic_load => {
let ty = fn_args.type_at(0);
let ordering = fn_args.const_at(1).to_value();
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization(ty);
return Ok(());
}
let layout = bx.layout_of(ty);
let source = args[0].immediate();
let llval = bx.atomic_load(
bx.backend_type(layout),
source,
parse_const_generic_ordering(ordering),
layout.size,
);
return ret_llval(bx, llval);
}
// The rest falls back to below.
_ => {}
sym::atomic_load => {
let ty = fn_args.type_at(0);
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization_int_type(ty);
return Ok(());
}
let Some((instruction, ordering)) = atomic.split_once('_') else {
bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
};
let parse_ordering = |bx: &Bx, s| match s {
"relaxed" => Relaxed,
"acquire" => Acquire,
"release" => Release,
"acqrel" => AcqRel,
"seqcst" => SeqCst,
_ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
};
match instruction {
"cxchg" | "cxchgweak" => {
let Some((success, failure)) = ordering.split_once('_') else {
bx.sess().dcx().emit_fatal(errors::AtomicCompareExchange);
};
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let weak = instruction == "cxchgweak";
let dst = args[0].immediate();
let cmp = args[1].immediate();
let src = args[2].immediate();
let (val, success) = bx.atomic_cmpxchg(
dst,
cmp,
src,
parse_ordering(bx, success),
parse_ordering(bx, failure),
weak,
);
let val = bx.from_immediate(val);
let success = bx.from_immediate(success);
let dest = result.project_field(bx, 0);
bx.store_to_place(val, dest.val);
let dest = result.project_field(bx, 1);
bx.store_to_place(success, dest.val);
} else {
invalid_monomorphization(ty);
}
return Ok(());
}
"store" => {
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let size = bx.layout_of(ty).size;
let val = args[1].immediate();
let ptr = args[0].immediate();
bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
} else {
invalid_monomorphization(ty);
}
return Ok(());
}
"fence" => {
bx.atomic_fence(
parse_ordering(bx, ordering),
SynchronizationScope::CrossThread,
);
return Ok(());
}
"singlethreadfence" => {
bx.atomic_fence(
parse_ordering(bx, ordering),
SynchronizationScope::SingleThread,
);
return Ok(());
}
// These are all AtomicRMW ops
"max" | "min" => {
let atom_op = if instruction == "max" {
AtomicRmwBinOp::AtomicMax
} else {
AtomicRmwBinOp::AtomicMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Int(_)) {
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
"umax" | "umin" => {
let atom_op = if instruction == "umax" {
AtomicRmwBinOp::AtomicUMax
} else {
AtomicRmwBinOp::AtomicUMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Uint(_)) {
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
op => {
let atom_op = match op {
"xchg" => AtomicRmwBinOp::AtomicXchg,
"xadd" => AtomicRmwBinOp::AtomicAdd,
"xsub" => AtomicRmwBinOp::AtomicSub,
"and" => AtomicRmwBinOp::AtomicAnd,
"nand" => AtomicRmwBinOp::AtomicNand,
"or" => AtomicRmwBinOp::AtomicOr,
"xor" => AtomicRmwBinOp::AtomicXor,
_ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOperation),
};
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
let ordering = fn_args.const_at(1).to_value();
let layout = bx.layout_of(ty);
let source = args[0].immediate();
bx.atomic_load(
bx.backend_type(layout),
source,
parse_atomic_ordering(ordering),
layout.size,
)
}
sym::atomic_store => {
let ty = fn_args.type_at(0);
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization_int_type(ty);
return Ok(());
}
let ordering = fn_args.const_at(1).to_value();
let size = bx.layout_of(ty).size;
let val = args[1].immediate();
let ptr = args[0].immediate();
bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size);
return Ok(());
}
sym::atomic_cxchg | sym::atomic_cxchgweak => {
let ty = fn_args.type_at(0);
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization_int_type(ty);
return Ok(());
}
let succ_ordering = fn_args.const_at(1).to_value();
let fail_ordering = fn_args.const_at(2).to_value();
let weak = name == sym::atomic_cxchgweak;
let dst = args[0].immediate();
let cmp = args[1].immediate();
let src = args[2].immediate();
let (val, success) = bx.atomic_cmpxchg(
dst,
cmp,
src,
parse_atomic_ordering(succ_ordering),
parse_atomic_ordering(fail_ordering),
weak,
);
let val = bx.from_immediate(val);
let success = bx.from_immediate(success);
let dest = result.project_field(bx, 0);
bx.store_to_place(val, dest.val);
let dest = result.project_field(bx, 1);
bx.store_to_place(success, dest.val);
return Ok(());
}
// These are all AtomicRMW ops
sym::atomic_max | sym::atomic_min => {
let atom_op = if name == sym::atomic_max {
AtomicRmwBinOp::AtomicMax
} else {
AtomicRmwBinOp::AtomicMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Int(_)) {
let ordering = fn_args.const_at(1).to_value();
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
} else {
invalid_monomorphization_int_type(ty);
return Ok(());
}
}
sym::atomic_umax | sym::atomic_umin => {
let atom_op = if name == sym::atomic_umax {
AtomicRmwBinOp::AtomicUMax
} else {
AtomicRmwBinOp::AtomicUMin
};
let ty = fn_args.type_at(0);
if matches!(ty.kind(), ty::Uint(_)) {
let ordering = fn_args.const_at(1).to_value();
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
} else {
invalid_monomorphization_int_type(ty);
return Ok(());
}
}
sym::atomic_xchg
| sym::atomic_xadd
| sym::atomic_xsub
| sym::atomic_and
| sym::atomic_nand
| sym::atomic_or
| sym::atomic_xor => {
let atom_op = match name {
sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg,
sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd,
sym::atomic_xsub => AtomicRmwBinOp::AtomicSub,
sym::atomic_and => AtomicRmwBinOp::AtomicAnd,
sym::atomic_nand => AtomicRmwBinOp::AtomicNand,
sym::atomic_or => AtomicRmwBinOp::AtomicOr,
sym::atomic_xor => AtomicRmwBinOp::AtomicXor,
_ => unreachable!(),
};
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let ordering = fn_args.const_at(1).to_value();
let ptr = args[0].immediate();
let val = args[1].immediate();
bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
} else {
invalid_monomorphization_int_type(ty);
return Ok(());
}
}
sym::atomic_fence => {
let ordering = fn_args.const_at(0).to_value();
bx.atomic_fence(parse_atomic_ordering(ordering), SynchronizationScope::CrossThread);
return Ok(());
}
sym::atomic_singlethreadfence => {
let ordering = fn_args.const_at(0).to_value();
bx.atomic_fence(
parse_atomic_ordering(ordering),
SynchronizationScope::SingleThread,
);
return Ok(());
}
sym::nontemporal_store => {
@@ -556,7 +516,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
};
ret_llval(bx, llval)
if result.layout.ty.is_bool() {
let val = bx.from_immediate(llval);
bx.store_to_place(val, result.val);
} else if !result.layout.ty.is_unit() {
bx.store_to_place(llval, result.val);
}
Ok(())
}
}