rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Rollup merge of #141507 - RalfJung:atomic-intrinsics, r=bjorn3

Browse Source 
atomic_load intrinsic: use const generic parameter for ordering

We have a gazillion intrinsics for the atomics because we encode the ordering into the intrinsic name rather than making it a parameter. This is particularly bad for those operations that take two orderings. Let's fix that!

This PR only converts `load`, to see if there's any feedback that would fundamentally change the strategy we pursue for the const generic intrinsics.

The first two commits are preparation and could be a separate PR if you prefer.

`@BoxyUwU` -- I hope this is a use of const generics that is unlikely to explode? All we need is a const generic of enum type. We could funnel it through an integer if we had to but an enum is obviously nicer...

`@bjorn3` it seems like the cranelift backend entirely ignores the ordering?
This commit is contained in:
Matthias Krüger
2025-05-30 07:01:30 +02:00
committed by GitHub
parent a87bc9d9fe a387c86a92
commit ad2d91ce11
20 changed files with 210 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
compiler/rustc_codegen_ssa/src/common.rs
View File

@@ -59,15 +59,6 @@ pub enum AtomicRmwBinOp {
AtomicUMin,
}
#[derive(Copy, Clone, Debug)]
pub enum AtomicOrdering {
Relaxed,
Acquire,
Release,
AcquireRelease,
SequentiallyConsistent,
}
#[derive(Copy, Clone, Debug)]
pub enum SynchronizationScope {
SingleThread,

91
compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
View File

@@ -99,6 +99,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
let ret_llval = |bx: &mut Bx, llval| {
if result.layout.ty.is_bool() {
OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
.val
.store(bx, result);
} else if !result.layout.ty.is_unit() {
bx.store_to_place(llval, result.val);
}
Ok(())
};
let llval = match name {
sym::abort => {
bx.abort();
@@ -334,9 +345,48 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// This requires that atomic intrinsics follow a specific naming pattern:
// "atomic_<operation>[_<ordering>]"
name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
use crate::common::AtomicOrdering::*;
use rustc_middle::ty::AtomicOrdering::*;
use crate::common::{AtomicRmwBinOp, SynchronizationScope};
let invalid_monomorphization = |ty| {
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
span,
name,
ty,
});
};
let parse_const_generic_ordering = |ord: ty::Value<'tcx>| {
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
discr.to_atomic_ordering()
};
// Some intrinsics have the ordering already converted to a const generic parameter, we handle those first.
match name {
sym::atomic_load => {
let ty = fn_args.type_at(0);
let ordering = fn_args.const_at(1).to_value();
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
invalid_monomorphization(ty);
return Ok(());
}
let layout = bx.layout_of(ty);
let source = args[0].immediate();
let llval = bx.atomic_load(
bx.backend_type(layout),
source,
parse_const_generic_ordering(ordering),
layout.size,
);
return ret_llval(bx, llval);
}
// The rest falls back to below.
_ => {}
}
let Some((instruction, ordering)) = atomic.split_once('_') else {
bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
};
@@ -345,19 +395,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
"relaxed" => Relaxed,
"acquire" => Acquire,
"release" => Release,
"acqrel" => AcquireRelease,
"seqcst" => SequentiallyConsistent,
"acqrel" => AcqRel,
"seqcst" => SeqCst,
_ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
};
let invalid_monomorphization = |ty| {
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
span,
name,
ty,
});
};
match instruction {
"cxchg" | "cxchgweak" => {
let Some((success, failure)) = ordering.split_once('_') else {
@@ -390,24 +432,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
return Ok(());
}
"load" => {
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
let layout = bx.layout_of(ty);
let size = layout.size;
let source = args[0].immediate();
bx.atomic_load(
bx.backend_type(layout),
source,
parse_ordering(bx, ordering),
size,
)
} else {
invalid_monomorphization(ty);
return Ok(());
}
}
"store" => {
let ty = fn_args.type_at(0);
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
@@ -538,14 +562,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
}
};
if result.layout.ty.is_bool() {
OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
.val
.store(bx, result);
} else if !result.layout.ty.is_unit() {
bx.store_to_place(llval, result.val);
}
Ok(())
ret_llval(bx, llval)
}
}

6
compiler/rustc_codegen_ssa/src/traits/builder.rs
View File

@@ -4,7 +4,7 @@ use std::ops::Deref;
use rustc_abi::{Align, Scalar, Size, WrappingRange};
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
use rustc_middle::ty::{Instance, Ty};
use rustc_middle::ty::{AtomicOrdering, Instance, Ty};
use rustc_session::config::OptLevel;
use rustc_span::Span;
use rustc_target::callconv::FnAbi;
@@ -19,9 +19,7 @@ use super::misc::MiscCodegenMethods;
use super::type_::{ArgAbiBuilderMethods, BaseTypeCodegenMethods, LayoutTypeCodegenMethods};
use super::{CodegenMethods, StaticBuilderMethods};
use crate::MemFlags;
use crate::common::{
AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
};
use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
use crate::mir::operand::{OperandRef, OperandValue};
use crate::mir::place::{PlaceRef, PlaceValue};