rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Simplify implementation of Rust intrinsics by using type parameters in the cache

Browse Source
This commit is contained in:
sayantn
2025-05-30 21:41:36 +05:30
parent 14863ea077
commit d56fcd968d
8 changed files with 450 additions and 800 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
compiler/rustc_codegen_llvm/src/builder.rs
View File

@@ -14,7 +14,6 @@ use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::*;
use rustc_data_structures::small_c_str::SmallCStr;
use rustc_hir::def_id::DefId;
use rustc_middle::bug;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
@@ -484,73 +483,31 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
fn checked_binop(
&mut self,
oop: OverflowOp,
ty: Ty<'_>,
ty: Ty<'tcx>,
lhs: Self::Value,
rhs: Self::Value,
) -> (Self::Value, Self::Value) {
use rustc_middle::ty::IntTy::*;
use rustc_middle::ty::UintTy::*;
use rustc_middle::ty::{Int, Uint};
let (size, signed) = ty.int_size_and_signed(self.tcx);
let width = size.bits();
let new_kind = match ty.kind() {
Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
t @ (Uint(_) | Int(_)) => *t,
_ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
if oop == OverflowOp::Sub && !signed {
// Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
// to be the canonical form. It will attempt to reform llvm.usub.with.overflow
// in the backend if profitable.
let sub = self.sub(lhs, rhs);
let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
return (sub, cmp);
}
let oop_str = match oop {
OverflowOp::Add => "add",
OverflowOp::Sub => "sub",
OverflowOp::Mul => "mul",
};
let name = match oop {
OverflowOp::Add => match new_kind {
Int(I8) => "llvm.sadd.with.overflow.i8",
Int(I16) => "llvm.sadd.with.overflow.i16",
Int(I32) => "llvm.sadd.with.overflow.i32",
Int(I64) => "llvm.sadd.with.overflow.i64",
Int(I128) => "llvm.sadd.with.overflow.i128",
let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
Uint(U8) => "llvm.uadd.with.overflow.i8",
Uint(U16) => "llvm.uadd.with.overflow.i16",
Uint(U32) => "llvm.uadd.with.overflow.i32",
Uint(U64) => "llvm.uadd.with.overflow.i64",
Uint(U128) => "llvm.uadd.with.overflow.i128",
_ => unreachable!(),
},
OverflowOp::Sub => match new_kind {
Int(I8) => "llvm.ssub.with.overflow.i8",
Int(I16) => "llvm.ssub.with.overflow.i16",
Int(I32) => "llvm.ssub.with.overflow.i32",
Int(I64) => "llvm.ssub.with.overflow.i64",
Int(I128) => "llvm.ssub.with.overflow.i128",
Uint(_) => {
// Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
// to be the canonical form. It will attempt to reform llvm.usub.with.overflow
// in the backend if profitable.
let sub = self.sub(lhs, rhs);
let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
return (sub, cmp);
}
_ => unreachable!(),
},
OverflowOp::Mul => match new_kind {
Int(I8) => "llvm.smul.with.overflow.i8",
Int(I16) => "llvm.smul.with.overflow.i16",
Int(I32) => "llvm.smul.with.overflow.i32",
Int(I64) => "llvm.smul.with.overflow.i64",
Int(I128) => "llvm.smul.with.overflow.i128",
Uint(U8) => "llvm.umul.with.overflow.i8",
Uint(U16) => "llvm.umul.with.overflow.i16",
Uint(U32) => "llvm.umul.with.overflow.i32",
Uint(U64) => "llvm.umul.with.overflow.i64",
Uint(U128) => "llvm.umul.with.overflow.i128",
_ => unreachable!(),
},
};
let res = self.call_intrinsic(name, &[lhs, rhs]);
let res = self.call_intrinsic(&name, &[self.type_ix(width)], &[lhs, rhs]);
(self.extract_value(res, 0), self.extract_value(res, 1))
}
@@ -954,11 +911,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
}
fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
self.fptoint_sat(false, val, dest_ty)
self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
}
fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
self.fptoint_sat(true, val, dest_ty)
self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
}
fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
@@ -981,15 +938,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
if self.cx.type_kind(src_ty) != TypeKind::Vector {
let float_width = self.cx.float_width(src_ty);
let int_width = self.cx.int_width(dest_ty);
let name = match (int_width, float_width) {
(32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
(32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
(64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
(64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
_ => None,
};
if let Some(name) = name {
return self.call_intrinsic(name, &[val]);
if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
return self.call_intrinsic(
"llvm.wasm.trunc.unsigned",
&[dest_ty, src_ty],
&[val],
);
}
}
}
@@ -1003,15 +957,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
if self.cx.type_kind(src_ty) != TypeKind::Vector {
let float_width = self.cx.float_width(src_ty);
let int_width = self.cx.int_width(dest_ty);
let name = match (int_width, float_width) {
(32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
(32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
(64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
(64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
_ => None,
};
if let Some(name) = name {
return self.call_intrinsic(name, &[val]);
if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
return self.call_intrinsic(
"llvm.wasm.trunc.signed",
&[dest_ty, src_ty],
&[val],
);
}
}
}
@@ -1084,22 +1035,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
return None;
}
let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) {
(true, 8) => "llvm.scmp.i8.i8",
(true, 16) => "llvm.scmp.i8.i16",
(true, 32) => "llvm.scmp.i8.i32",
(true, 64) => "llvm.scmp.i8.i64",
(true, 128) => "llvm.scmp.i8.i128",
let size = ty.primitive_size(self.tcx);
let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
(false, 8) => "llvm.ucmp.i8.i8",
(false, 16) => "llvm.ucmp.i8.i16",
(false, 32) => "llvm.ucmp.i8.i32",
(false, 64) => "llvm.ucmp.i8.i64",
(false, 128) => "llvm.ucmp.i8.i128",
_ => bug!("three-way compare unsupported for type {ty:?}"),
};
Some(self.call_intrinsic(name, &[lhs, rhs]))
Some(self.call_intrinsic(&name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
}
/* Miscellaneous instructions */
@@ -1385,11 +1324,11 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
}
fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
}
fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
}
fn call(
@@ -1454,7 +1393,7 @@ impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
// Forward to the `get_static` method of `CodegenCx`
let global = self.cx().get_static(def_id);
if self.cx().tcx.is_thread_local_static(def_id) {
let pointer = self.call_intrinsic("llvm.threadlocal.address", &[global]);
let pointer = self.call_intrinsic("llvm.threadlocal.address", &[], &[global]);
// Cast to default address space if globals are in a different addrspace
self.pointercast(pointer, self.type_ptr())
} else {
@@ -1649,8 +1588,13 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
}
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
let (ty, f) = self.cx.get_intrinsic(intrinsic);
pub(crate) fn call_intrinsic(
&mut self,
base_name: &str,
type_params: &[&'ll Type],
args: &[&'ll Value],
) -> &'ll Value {
let (ty, f) = self.cx.get_intrinsic(base_name, type_params);
self.call(ty, None, None, f, args, None, None)
}
@@ -1664,7 +1608,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
return;
}
self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
self.call_intrinsic(intrinsic, &[self.type_ptr()], &[self.cx.const_u64(size), ptr]);
}
}
impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
@@ -1689,31 +1633,6 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
}
}
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
let src_ty = self.cx.val_ty(val);
let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
(
self.cx.element_type(src_ty),
self.cx.element_type(dest_ty),
Some(self.cx.vector_length(src_ty)),
)
} else {
(src_ty, dest_ty, None)
};
let float_width = self.cx.float_width(float_ty);
let int_width = self.cx.int_width(int_ty);
let instr = if signed { "fptosi" } else { "fptoui" };
let name = if let Some(vector_length) = vector_length {
format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}")
} else {
format!("llvm.{instr}.sat.i{int_width}.f{float_width}")
};
let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None)
}
pub(crate) fn landing_pad(
&mut self,
ty: &'ll Type,
@@ -1819,7 +1738,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
// llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
// calls to this intrinsic with code to test type membership.
let typeid = self.get_metadata_value(typeid_metadata);
let cond = self.call_intrinsic("llvm.type.test", &[llfn, typeid]);
let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
let bb_pass = self.append_sibling_block("type_test.pass");
let bb_fail = self.append_sibling_block("type_test.fail");
self.cond_br(cond, bb_pass, bb_fail);
@@ -1887,7 +1806,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
num_counters: &'ll Value,
index: &'ll Value,
) {
self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]);
self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
}
/// Emits a call to `llvm.instrprof.mcdc.parameters`.
@@ -1906,7 +1825,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
hash: &'ll Value,
bitmap_bits: &'ll Value,
) {
self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]);
self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]);
}
#[instrument(level = "debug", skip(self))]
@@ -1918,7 +1837,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
mcdc_temp: &'ll Value,
) {
let args = &[fn_name, hash, bitmap_index, mcdc_temp];
self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args);
self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args);
}
#[instrument(level = "debug", skip(self))]

448
compiler/rustc_codegen_llvm/src/context.rs
View File

@@ -137,7 +137,8 @@ pub(crate) struct FullCx<'ll, 'tcx> {
eh_catch_typeinfo: Cell<Option<&'ll Value>>,
pub rust_try_fn: Cell<Option<(&'ll Type, &'ll Value)>>,
intrinsics: RefCell<FxHashMap<&'static str, (&'ll Type, &'ll Value)>>,
intrinsics:
RefCell<FxHashMap<(&'static str, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
/// A counter that is used for generating local symbol names
local_gen_sym_counter: Cell<usize>,
@@ -842,17 +843,24 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
}
impl<'ll> CodegenCx<'ll, '_> {
pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) {
if let Some(v) = self.intrinsics.borrow().get(key).cloned() {
return v;
pub(crate) fn get_intrinsic(
&self,
base_name: &str,
type_params: &[&'ll Type],
) -> (&'ll Type, &'ll Value) {
if let Some(v) =
self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params)))
{
return *v;
}
self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key))
self.declare_intrinsic(base_name, type_params)
}
fn insert_intrinsic(
&self,
name: &'static str,
base_name: &'static str,
type_params: &[&'ll Type],
args: Option<&[&'ll llvm::Type]>,
ret: &'ll llvm::Type,
) -> (&'ll llvm::Type, &'ll llvm::Value) {
@@ -861,372 +869,153 @@ impl<'ll> CodegenCx<'ll, '_> {
} else {
self.type_variadic_func(&[], ret)
};
let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty);
self.intrinsics.borrow_mut().insert(name, (fn_ty, f));
let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
.expect("Unknown LLVM intrinsic `{base_name}`");
let full_name = if intrinsic.is_overloaded() {
&intrinsic.overloaded_name(self.llmod, type_params)
} else {
base_name
};
let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty);
self.intrinsics
.borrow_mut()
.insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f));
(fn_ty, f)
}
fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> {
fn declare_intrinsic(
&self,
base_name: &str,
type_params: &[&'ll Type],
) -> (&'ll Type, &'ll Value) {
macro_rules! param {
($index:literal) => {
type_params[$index]
};
($other:expr) => {
$other
};
}
macro_rules! ifn {
($name:expr, fn() -> $ret:expr) => (
if key == $name {
return Some(self.insert_intrinsic($name, Some(&[]), $ret));
}
);
($name:expr, fn(...) -> $ret:expr) => (
if key == $name {
return Some(self.insert_intrinsic($name, None, $ret));
if base_name == $name {
return self.insert_intrinsic($name, type_params, None, param!($ret));
}
);
($name:expr, fn($($arg:expr),*) -> $ret:expr) => (
if key == $name {
return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret));
if base_name == $name {
return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret));
}
);
}
macro_rules! mk_struct {
($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false))
($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false))
}
let same_width_vector = |index, element_ty| {
self.type_vector(element_ty, self.vector_length(type_params[index]) as u64)
};
let ptr = self.type_ptr();
let void = self.type_void();
let i1 = self.type_i1();
let t_i8 = self.type_i8();
let t_i16 = self.type_i16();
let t_i32 = self.type_i32();
let t_i64 = self.type_i64();
let t_i128 = self.type_i128();
let t_isize = self.type_isize();
let t_f16 = self.type_f16();
let t_f32 = self.type_f32();
let t_f64 = self.type_f64();
let t_f128 = self.type_f128();
let t_metadata = self.type_metadata();
let t_token = self.type_token();
ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr);
ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32);
ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64);
ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64);
ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32);
ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32);
ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64);
ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64);
ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8);
ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16);
ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32);
ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64);
ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128);
ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8);
ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16);
ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32);
ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64);
ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128);
ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8);
ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16);
ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32);
ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64);
ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128);
ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8);
ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16);
ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32);
ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64);
ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128);
ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0);
ifn!("llvm.wasm.trunc.signed", fn(1) -> 0);
ifn!("llvm.fptosi.sat", fn(1) -> 0);
ifn!("llvm.fptoui.sat", fn(1) -> 0);
ifn!("llvm.trap", fn() -> void);
ifn!("llvm.debugtrap", fn() -> void);
ifn!("llvm.frameaddress", fn(t_i32) -> ptr);
ifn!("llvm.powi.f16.i32", fn(t_f16, t_i32) -> t_f16);
ifn!("llvm.powi.f32.i32", fn(t_f32, t_i32) -> t_f32);
ifn!("llvm.powi.f64.i32", fn(t_f64, t_i32) -> t_f64);
ifn!("llvm.powi.f128.i32", fn(t_f128, t_i32) -> t_f128);
ifn!("llvm.powi", fn(0, 1) -> 0);
ifn!("llvm.pow", fn(0, 0) -> 0);
ifn!("llvm.sqrt", fn(0) -> 0);
ifn!("llvm.sin", fn(0) -> 0);
ifn!("llvm.cos", fn(0) -> 0);
ifn!("llvm.exp", fn(0) -> 0);
ifn!("llvm.exp2", fn(0) -> 0);
ifn!("llvm.log", fn(0) -> 0);
ifn!("llvm.log10", fn(0) -> 0);
ifn!("llvm.log2", fn(0) -> 0);
ifn!("llvm.fma", fn(0, 0, 0) -> 0);
ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0);
ifn!("llvm.fabs", fn(0) -> 0);
ifn!("llvm.minnum", fn(0, 0) -> 0);
ifn!("llvm.minimum", fn(0, 0) -> 0);
ifn!("llvm.maxnum", fn(0, 0) -> 0);
ifn!("llvm.maximum", fn(0, 0) -> 0);
ifn!("llvm.floor", fn(0) -> 0);
ifn!("llvm.ceil", fn(0) -> 0);
ifn!("llvm.trunc", fn(0) -> 0);
ifn!("llvm.copysign", fn(0, 0) -> 0);
ifn!("llvm.round", fn(0) -> 0);
ifn!("llvm.rint", fn(0) -> 0);
ifn!("llvm.nearbyint", fn(0) -> 0);
ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16);
ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64);
ifn!("llvm.pow.f128", fn(t_f128, t_f128) -> t_f128);
ifn!("llvm.ctpop", fn(0) -> 0);
ifn!("llvm.ctlz", fn(0, i1) -> 0);
ifn!("llvm.cttz", fn(0, i1) -> 0);
ifn!("llvm.bswap", fn(0) -> 0);
ifn!("llvm.bitreverse", fn(0) -> 0);
ifn!("llvm.fshl", fn(0, 0, 0) -> 0);
ifn!("llvm.fshr", fn(0, 0, 0) -> 0);
ifn!("llvm.sqrt.f16", fn(t_f16) -> t_f16);
ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32);
ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64);
ifn!("llvm.sqrt.f128", fn(t_f128) -> t_f128);
ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.sin.f16", fn(t_f16) -> t_f16);
ifn!("llvm.sin.f32", fn(t_f32) -> t_f32);
ifn!("llvm.sin.f64", fn(t_f64) -> t_f64);
ifn!("llvm.sin.f128", fn(t_f128) -> t_f128);
ifn!("llvm.sadd.sat", fn(0, 0) -> 0);
ifn!("llvm.uadd.sat", fn(0, 0) -> 0);
ifn!("llvm.ssub.sat", fn(0, 0) -> 0);
ifn!("llvm.usub.sat", fn(0, 0) -> 0);
ifn!("llvm.cos.f16", fn(t_f16) -> t_f16);
ifn!("llvm.cos.f32", fn(t_f32) -> t_f32);
ifn!("llvm.cos.f64", fn(t_f64) -> t_f64);
ifn!("llvm.cos.f128", fn(t_f128) -> t_f128);
ifn!("llvm.scmp", fn(1, 1) -> 0);
ifn!("llvm.ucmp", fn(1, 1) -> 0);
ifn!("llvm.exp.f16", fn(t_f16) -> t_f16);
ifn!("llvm.exp.f32", fn(t_f32) -> t_f32);
ifn!("llvm.exp.f64", fn(t_f64) -> t_f64);
ifn!("llvm.exp.f128", fn(t_f128) -> t_f128);
ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void);
ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void);
ifn!("llvm.exp2.f16", fn(t_f16) -> t_f16);
ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32);
ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64);
ifn!("llvm.exp2.f128", fn(t_f128) -> t_f128);
ifn!("llvm.is.constant", fn(0) -> i1);
ifn!("llvm.expect", fn(0, 0) -> 0);
ifn!("llvm.log.f16", fn(t_f16) -> t_f16);
ifn!("llvm.log.f32", fn(t_f32) -> t_f32);
ifn!("llvm.log.f64", fn(t_f64) -> t_f64);
ifn!("llvm.log.f128", fn(t_f128) -> t_f128);
ifn!("llvm.log10.f16", fn(t_f16) -> t_f16);
ifn!("llvm.log10.f32", fn(t_f32) -> t_f32);
ifn!("llvm.log10.f64", fn(t_f64) -> t_f64);
ifn!("llvm.log10.f128", fn(t_f128) -> t_f128);
ifn!("llvm.log2.f16", fn(t_f16) -> t_f16);
ifn!("llvm.log2.f32", fn(t_f32) -> t_f32);
ifn!("llvm.log2.f64", fn(t_f64) -> t_f64);
ifn!("llvm.log2.f128", fn(t_f128) -> t_f128);
ifn!("llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16);
ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32);
ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64);
ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128);
ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16);
ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32);
ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64);
ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128);
ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16);
ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32);
ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64);
ifn!("llvm.fabs.f128", fn(t_f128) -> t_f128);
ifn!("llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16);
ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64);
ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128);
ifn!("llvm.minimum.f16", fn(t_f16, t_f16) -> t_f16);
ifn!("llvm.minimum.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.minimum.f64", fn(t_f64, t_f64) -> t_f64);
// There are issues on x86_64 and aarch64 with the f128 variant.
// - https://github.com/llvm/llvm-project/issues/139380
// - https://github.com/llvm/llvm-project/issues/139381
// ifn!("llvm.minimum.f128", fn(t_f128, t_f128) -> t_f128);
ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16);
ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64);
ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128);
ifn!("llvm.maximum.f16", fn(t_f16, t_f16) -> t_f16);
ifn!("llvm.maximum.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.maximum.f64", fn(t_f64, t_f64) -> t_f64);
// There are issues on x86_64 and aarch64 with the f128 variant.
// - https://github.com/llvm/llvm-project/issues/139380
// - https://github.com/llvm/llvm-project/issues/139381
// ifn!("llvm.maximum.f128", fn(t_f128, t_f128) -> t_f128);
ifn!("llvm.floor.f16", fn(t_f16) -> t_f16);
ifn!("llvm.floor.f32", fn(t_f32) -> t_f32);
ifn!("llvm.floor.f64", fn(t_f64) -> t_f64);
ifn!("llvm.floor.f128", fn(t_f128) -> t_f128);
ifn!("llvm.ceil.f16", fn(t_f16) -> t_f16);
ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32);
ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64);
ifn!("llvm.ceil.f128", fn(t_f128) -> t_f128);
ifn!("llvm.trunc.f16", fn(t_f16) -> t_f16);
ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32);
ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64);
ifn!("llvm.trunc.f128", fn(t_f128) -> t_f128);
ifn!("llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16);
ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32);
ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64);
ifn!("llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128);
ifn!("llvm.round.f16", fn(t_f16) -> t_f16);
ifn!("llvm.round.f32", fn(t_f32) -> t_f32);
ifn!("llvm.round.f64", fn(t_f64) -> t_f64);
ifn!("llvm.round.f128", fn(t_f128) -> t_f128);
ifn!("llvm.roundeven.f16", fn(t_f16) -> t_f16);
ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32);
ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64);
ifn!("llvm.roundeven.f128", fn(t_f128) -> t_f128);
ifn!("llvm.rint.f16", fn(t_f16) -> t_f16);
ifn!("llvm.rint.f32", fn(t_f32) -> t_f32);
ifn!("llvm.rint.f64", fn(t_f64) -> t_f64);
ifn!("llvm.rint.f128", fn(t_f128) -> t_f128);
ifn!("llvm.nearbyint.f16", fn(t_f16) -> t_f16);
ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32);
ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64);
ifn!("llvm.nearbyint.f128", fn(t_f128) -> t_f128);
ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8);
ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16);
ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32);
ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64);
ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128);
ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8);
ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16);
ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32);
ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64);
ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128);
ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8);
ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16);
ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32);
ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64);
ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128);
ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16);
ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32);
ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64);
ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128);
ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8);
ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16);
ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32);
ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64);
ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128);
ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8);
ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16);
ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32);
ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64);
ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128);
ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8);
ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16);
ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32);
ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64);
ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128);
ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16);
ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32);
ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64);
ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128);
ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16);
ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32);
ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64);
ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128);
ifn!("llvm.scmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.scmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
ifn!("llvm.scmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
ifn!("llvm.scmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
ifn!("llvm.scmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
ifn!("llvm.ucmp.i8.i8", fn(t_i8, t_i8) -> t_i8);
ifn!("llvm.ucmp.i8.i16", fn(t_i16, t_i16) -> t_i8);
ifn!("llvm.ucmp.i8.i32", fn(t_i32, t_i32) -> t_i8);
ifn!("llvm.ucmp.i8.i64", fn(t_i64, t_i64) -> t_i8);
ifn!("llvm.ucmp.i8.i128", fn(t_i128, t_i128) -> t_i8);
ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void);
ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void);
// FIXME: This is an infinitesimally small portion of the types you can
// pass to this intrinsic, if we can ever lazily register intrinsics we
// should register these when they're used, that way any type can be
// passed.
ifn!("llvm.is.constant.i1", fn(i1) -> i1);
ifn!("llvm.is.constant.i8", fn(t_i8) -> i1);
ifn!("llvm.is.constant.i16", fn(t_i16) -> i1);
ifn!("llvm.is.constant.i32", fn(t_i32) -> i1);
ifn!("llvm.is.constant.i64", fn(t_i64) -> i1);
ifn!("llvm.is.constant.i128", fn(t_i128) -> i1);
ifn!("llvm.is.constant.isize", fn(t_isize) -> i1);
ifn!("llvm.is.constant.f16", fn(t_f16) -> i1);
ifn!("llvm.is.constant.f32", fn(t_f32) -> i1);
ifn!("llvm.is.constant.f64", fn(t_f64) -> i1);
ifn!("llvm.is.constant.f128", fn(t_f128) -> i1);
ifn!("llvm.is.constant.ptr", fn(ptr) -> i1);
ifn!("llvm.expect.i1", fn(i1, i1) -> i1);
ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32);
ifn!("llvm.eh.typeid.for", fn(0) -> t_i32);
ifn!("llvm.localescape", fn(...) -> void);
ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr);
ifn!("llvm.x86.seh.recoverfp", fn(ptr, ptr) -> ptr);
ifn!("llvm.assume", fn(i1) -> void);
ifn!("llvm.prefetch", fn(ptr, t_i32, t_i32, t_i32) -> void);
ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void);
// This isn't an "LLVM intrinsic", but LLVM's optimization passes
// recognize it like one (including turning it into `bcmp` sometimes)
// and we use it to implement intrinsics like `raw_eq` and `compare_bytes`
match self.sess().target.arch.as_ref() {
"avr" | "msp430" => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i16),
_ => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i32),
if base_name == "memcmp" {
let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int());
let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty);
self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f));
return (fn_ty, f);
}
// variadic intrinsics
ifn!("llvm.va_start", fn(ptr) -> void);
ifn!("llvm.va_end", fn(ptr) -> void);
ifn!("llvm.va_copy", fn(ptr, ptr) -> void);
ifn!("llvm.va_start", fn(0) -> void);
ifn!("llvm.va_end", fn(0) -> void);
ifn!("llvm.va_copy", fn(0, 0) -> void);
if self.sess().instrument_coverage() {
ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
@@ -1238,14 +1027,19 @@ impl<'ll> CodegenCx<'ll, '_> {
ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1});
if self.sess().opts.debuginfo != DebugInfo::None {
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void);
ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void);
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void);
ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void);
}
ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr);
ifn!("llvm.ptrmask", fn(0, 1) -> 0);
ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);
None
ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
bug!("Unknown intrinsic: `{base_name}`")
}
pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value {

549
compiler/rustc_codegen_llvm/src/intrinsic.rs
View File

@@ -15,7 +15,7 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty};
use rustc_middle::{bug, span_bug};
use rustc_span::{Span, Symbol, sym};
use rustc_symbol_mangling::mangle_internal_symbol;
use rustc_target::spec::{HasTargetSpec, PanicStrategy};
use rustc_target::spec::PanicStrategy;
use tracing::debug;
use crate::abi::FnAbiLlvmExt;
@@ -27,137 +27,142 @@ use crate::type_of::LayoutLlvmExt;
use crate::va_arg::emit_va_arg;
use crate::value::Value;
fn get_simple_intrinsic<'ll>(
cx: &CodegenCx<'ll, '_>,
fn call_simple_intrinsic<'ll, 'tcx>(
bx: &mut Builder<'_, 'll, 'tcx>,
name: Symbol,
) -> Option<(&'ll Type, &'ll Value)> {
let llvm_name = match name {
sym::sqrtf16 => "llvm.sqrt.f16",
sym::sqrtf32 => "llvm.sqrt.f32",
sym::sqrtf64 => "llvm.sqrt.f64",
sym::sqrtf128 => "llvm.sqrt.f128",
args: &[OperandRef<'tcx, &'ll Value>],
) -> Option<&'ll Value> {
let (base_name, type_params): (&'static str, &[&'ll Type]) = match name {
sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]),
sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]),
sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]),
sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]),
sym::powif16 => "llvm.powi.f16.i32",
sym::powif32 => "llvm.powi.f32.i32",
sym::powif64 => "llvm.powi.f64.i32",
sym::powif128 => "llvm.powi.f128.i32",
sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]),
sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]),
sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]),
sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]),
sym::sinf16 => "llvm.sin.f16",
sym::sinf32 => "llvm.sin.f32",
sym::sinf64 => "llvm.sin.f64",
sym::sinf128 => "llvm.sin.f128",
sym::sinf16 => ("llvm.sin", &[bx.type_f16()]),
sym::sinf32 => ("llvm.sin", &[bx.type_f32()]),
sym::sinf64 => ("llvm.sin", &[bx.type_f64()]),
sym::sinf128 => ("llvm.sin", &[bx.type_f128()]),
sym::cosf16 => "llvm.cos.f16",
sym::cosf32 => "llvm.cos.f32",
sym::cosf64 => "llvm.cos.f64",
sym::cosf128 => "llvm.cos.f128",
sym::cosf16 => ("llvm.cos", &[bx.type_f16()]),
sym::cosf32 => ("llvm.cos", &[bx.type_f32()]),
sym::cosf64 => ("llvm.cos", &[bx.type_f64()]),
sym::cosf128 => ("llvm.cos", &[bx.type_f128()]),
sym::powf16 => "llvm.pow.f16",
sym::powf32 => "llvm.pow.f32",
sym::powf64 => "llvm.pow.f64",
sym::powf128 => "llvm.pow.f128",
sym::powf16 => ("llvm.pow", &[bx.type_f16()]),
sym::powf32 => ("llvm.pow", &[bx.type_f32()]),
sym::powf64 => ("llvm.pow", &[bx.type_f64()]),
sym::powf128 => ("llvm.pow", &[bx.type_f128()]),
sym::expf16 => "llvm.exp.f16",
sym::expf32 => "llvm.exp.f32",
sym::expf64 => "llvm.exp.f64",
sym::expf128 => "llvm.exp.f128",
sym::expf16 => ("llvm.exp", &[bx.type_f16()]),
sym::expf32 => ("llvm.exp", &[bx.type_f32()]),
sym::expf64 => ("llvm.exp", &[bx.type_f64()]),
sym::expf128 => ("llvm.exp", &[bx.type_f128()]),
sym::exp2f16 => "llvm.exp2.f16",
sym::exp2f32 => "llvm.exp2.f32",
sym::exp2f64 => "llvm.exp2.f64",
sym::exp2f128 => "llvm.exp2.f128",
sym::exp2f16 => ("llvm.exp2", &[bx.type_f16()]),
sym::exp2f32 => ("llvm.exp2", &[bx.type_f32()]),
sym::exp2f64 => ("llvm.exp2", &[bx.type_f64()]),
sym::exp2f128 => ("llvm.exp2", &[bx.type_f128()]),
sym::logf16 => "llvm.log.f16",
sym::logf32 => "llvm.log.f32",
sym::logf64 => "llvm.log.f64",
sym::logf128 => "llvm.log.f128",
sym::logf16 => ("llvm.log", &[bx.type_f16()]),
sym::logf32 => ("llvm.log", &[bx.type_f32()]),
sym::logf64 => ("llvm.log", &[bx.type_f64()]),
sym::logf128 => ("llvm.log", &[bx.type_f128()]),
sym::log10f16 => "llvm.log10.f16",
sym::log10f32 => "llvm.log10.f32",
sym::log10f64 => "llvm.log10.f64",
sym::log10f128 => "llvm.log10.f128",
sym::log10f16 => ("llvm.log10", &[bx.type_f16()]),
sym::log10f32 => ("llvm.log10", &[bx.type_f32()]),
sym::log10f64 => ("llvm.log10", &[bx.type_f64()]),
sym::log10f128 => ("llvm.log10", &[bx.type_f128()]),
sym::log2f16 => "llvm.log2.f16",
sym::log2f32 => "llvm.log2.f32",
sym::log2f64 => "llvm.log2.f64",
sym::log2f128 => "llvm.log2.f128",
sym::log2f16 => ("llvm.log2", &[bx.type_f16()]),
sym::log2f32 => ("llvm.log2", &[bx.type_f32()]),
sym::log2f64 => ("llvm.log2", &[bx.type_f64()]),
sym::log2f128 => ("llvm.log2", &[bx.type_f128()]),
sym::fmaf16 => "llvm.fma.f16",
sym::fmaf32 => "llvm.fma.f32",
sym::fmaf64 => "llvm.fma.f64",
sym::fmaf128 => "llvm.fma.f128",
sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]),
sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]),
sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]),
sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]),
sym::fmuladdf16 => "llvm.fmuladd.f16",
sym::fmuladdf32 => "llvm.fmuladd.f32",
sym::fmuladdf64 => "llvm.fmuladd.f64",
sym::fmuladdf128 => "llvm.fmuladd.f128",
sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]),
sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]),
sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]),
sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]),
sym::fabsf16 => "llvm.fabs.f16",
sym::fabsf32 => "llvm.fabs.f32",
sym::fabsf64 => "llvm.fabs.f64",
sym::fabsf128 => "llvm.fabs.f128",
sym::fabsf16 => ("llvm.fabs", &[bx.type_f16()]),
sym::fabsf32 => ("llvm.fabs", &[bx.type_f32()]),
sym::fabsf64 => ("llvm.fabs", &[bx.type_f64()]),
sym::fabsf128 => ("llvm.fabs", &[bx.type_f128()]),
sym::minnumf16 => "llvm.minnum.f16",
sym::minnumf32 => "llvm.minnum.f32",
sym::minnumf64 => "llvm.minnum.f64",
sym::minnumf128 => "llvm.minnum.f128",
sym::minnumf16 => ("llvm.minnum", &[bx.type_f16()]),
sym::minnumf32 => ("llvm.minnum", &[bx.type_f32()]),
sym::minnumf64 => ("llvm.minnum", &[bx.type_f64()]),
sym::minnumf128 => ("llvm.minnum", &[bx.type_f128()]),
sym::minimumf16 => "llvm.minimum.f16",
sym::minimumf32 => "llvm.minimum.f32",
sym::minimumf64 => "llvm.minimum.f64",
sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]),
sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]),
sym::minimumf64 => ("llvm.minimum", &[bx.type_f64()]),
// There are issues on x86_64 and aarch64 with the f128 variant,
// let's instead use the instrinsic fallback body.
// sym::minimumf128 => "llvm.minimum.f128",
sym::maxnumf16 => "llvm.maxnum.f16",
sym::maxnumf32 => "llvm.maxnum.f32",
sym::maxnumf64 => "llvm.maxnum.f64",
sym::maxnumf128 => "llvm.maxnum.f128",
// sym::minimumf128 => ("llvm.minimum", &[cx.type_f128()]),
sym::maxnumf16 => ("llvm.maxnum", &[bx.type_f16()]),
sym::maxnumf32 => ("llvm.maxnum", &[bx.type_f32()]),
sym::maxnumf64 => ("llvm.maxnum", &[bx.type_f64()]),
sym::maxnumf128 => ("llvm.maxnum", &[bx.type_f128()]),
sym::maximumf16 => "llvm.maximum.f16",
sym::maximumf32 => "llvm.maximum.f32",
sym::maximumf64 => "llvm.maximum.f64",
sym::maximumf16 => ("llvm.maximum", &[bx.type_f16()]),
sym::maximumf32 => ("llvm.maximum", &[bx.type_f32()]),
sym::maximumf64 => ("llvm.maximum", &[bx.type_f64()]),
// There are issues on x86_64 and aarch64 with the f128 variant,
// let's instead use the instrinsic fallback body.
// sym::maximumf128 => "llvm.maximum.f128",
sym::copysignf16 => "llvm.copysign.f16",
sym::copysignf32 => "llvm.copysign.f32",
sym::copysignf64 => "llvm.copysign.f64",
sym::copysignf128 => "llvm.copysign.f128",
// sym::maximumf128 => ("llvm.maximum", &[cx.type_f128()]),
sym::copysignf16 => ("llvm.copysign", &[bx.type_f16()]),
sym::copysignf32 => ("llvm.copysign", &[bx.type_f32()]),
sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]),
sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]),
sym::floorf16 => "llvm.floor.f16",
sym::floorf32 => "llvm.floor.f32",
sym::floorf64 => "llvm.floor.f64",
sym::floorf128 => "llvm.floor.f128",
sym::floorf16 => ("llvm.floor", &[bx.type_f16()]),
sym::floorf32 => ("llvm.floor", &[bx.type_f32()]),
sym::floorf64 => ("llvm.floor", &[bx.type_f64()]),
sym::floorf128 => ("llvm.floor", &[bx.type_f128()]),
sym::ceilf16 => "llvm.ceil.f16",
sym::ceilf32 => "llvm.ceil.f32",
sym::ceilf64 => "llvm.ceil.f64",
sym::ceilf128 => "llvm.ceil.f128",
sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]),
sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]),
sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]),
sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]),
sym::truncf16 => "llvm.trunc.f16",
sym::truncf32 => "llvm.trunc.f32",
sym::truncf64 => "llvm.trunc.f64",
sym::truncf128 => "llvm.trunc.f128",
sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]),
sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]),
sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]),
sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]),
// We could use any of `rint`, `nearbyint`, or `roundeven`
// for this -- they are all identical in semantics when
// assuming the default FP environment.
// `rint` is what we used for $forever.
sym::round_ties_even_f16 => "llvm.rint.f16",
sym::round_ties_even_f32 => "llvm.rint.f32",
sym::round_ties_even_f64 => "llvm.rint.f64",
sym::round_ties_even_f128 => "llvm.rint.f128",
sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]),
sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]),
sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]),
sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]),
sym::roundf16 => "llvm.round.f16",
sym::roundf32 => "llvm.round.f32",
sym::roundf64 => "llvm.round.f64",
sym::roundf128 => "llvm.round.f128",
sym::roundf16 => ("llvm.round", &[bx.type_f16()]),
sym::roundf32 => ("llvm.round", &[bx.type_f32()]),
sym::roundf64 => ("llvm.round", &[bx.type_f64()]),
sym::roundf128 => ("llvm.round", &[bx.type_f128()]),
sym::ptr_mask => "llvm.ptrmask",
sym::ptr_mask => ("llvm.ptrmask", &[bx.type_ptr(), bx.type_isize()]),
_ => return None,
};
Some(cx.get_intrinsic(llvm_name))
Some(bx.call_intrinsic(
base_name,
type_params,
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
))
}
impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
@@ -173,36 +178,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
let name = tcx.item_name(instance.def_id());
let fn_args = instance.args;
let simple = get_simple_intrinsic(self, name);
let simple = call_simple_intrinsic(self, name, args);
let llval = match name {
_ if simple.is_some() => {
let (simple_ty, simple_fn) = simple.unwrap();
self.call(
simple_ty,
None,
None,
simple_fn,
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
None,
Some(instance),
)
}
_ if simple.is_some() => simple.unwrap(),
sym::is_val_statically_known => {
let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
let kind = self.type_kind(intrinsic_type);
let intrinsic_name = match kind {
TypeKind::Pointer | TypeKind::Integer => {
Some(format!("llvm.is.constant.{intrinsic_type:?}"))
}
// LLVM float types' intrinsic names differ from their type names.
TypeKind::Half => Some(format!("llvm.is.constant.f16")),
TypeKind::Float => Some(format!("llvm.is.constant.f32")),
TypeKind::Double => Some(format!("llvm.is.constant.f64")),
TypeKind::FP128 => Some(format!("llvm.is.constant.f128")),
_ => None,
};
if let Some(intrinsic_name) = intrinsic_name {
self.call_intrinsic(&intrinsic_name, &[args[0].immediate()])
if let OperandValue::Immediate(imm) = args[0].val {
self.call_intrinsic(
"llvm.is.constant",
&[args[0].layout.immediate_llvm_type(self.cx)],
&[imm],
)
} else {
self.const_bool(false)
}
@@ -246,10 +231,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
);
return Ok(());
}
sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]),
sym::va_copy => {
self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
}
sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]),
sym::va_copy => self.call_intrinsic(
"llvm.va_copy",
&[self.type_ptr()],
&[args[0].immediate(), args[1].immediate()],
),
sym::va_arg => {
match result.layout.backend_repr {
BackendRepr::Scalar(scalar) => {
@@ -324,6 +311,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
};
self.call_intrinsic(
"llvm.prefetch",
&[self.type_ptr()],
&[
args[0].immediate(),
self.const_i32(rw),
@@ -385,11 +373,13 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
let (size, signed) = ty.int_size_and_signed(self.tcx);
let width = size.bits();
let llty = self.type_ix(width);
match name {
sym::ctlz | sym::cttz => {
let y = self.const_bool(false);
let ret = self.call_intrinsic(
&format!("llvm.{name}.i{width}"),
&format!("llvm.{name}"),
&[llty],
&[args[0].immediate(), y],
);
@@ -397,62 +387,54 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
sym::ctlz_nonzero => {
let y = self.const_bool(true);
let llvm_name = &format!("llvm.ctlz.i{width}");
let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
let ret =
self.call_intrinsic("llvm.ctlz", &[llty], &[args[0].immediate(), y]);
self.intcast(ret, result.layout.llvm_type(self), false)
}
sym::cttz_nonzero => {
let y = self.const_bool(true);
let llvm_name = &format!("llvm.cttz.i{width}");
let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
let ret =
self.call_intrinsic("llvm.cttz", &[llty], &[args[0].immediate(), y]);
self.intcast(ret, result.layout.llvm_type(self), false)
}
sym::ctpop => {
let ret = self.call_intrinsic(
&format!("llvm.ctpop.i{width}"),
&[args[0].immediate()],
);
let ret =
self.call_intrinsic("llvm.ctpop", &[llty], &[args[0].immediate()]);
self.intcast(ret, result.layout.llvm_type(self), false)
}
sym::bswap => {
if width == 8 {
args[0].immediate() // byte swap a u8/i8 is just a no-op
} else {
self.call_intrinsic(
&format!("llvm.bswap.i{width}"),
&[args[0].immediate()],
)
self.call_intrinsic("llvm.bswap", &[llty], &[args[0].immediate()])
}
}
sym::bitreverse => self.call_intrinsic(
&format!("llvm.bitreverse.i{width}"),
&[args[0].immediate()],
),
sym::bitreverse => {
self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
}
sym::rotate_left | sym::rotate_right => {
let is_left = name == sym::rotate_left;
let val = args[0].immediate();
let raw_shift = args[1].immediate();
// rotate = funnel shift with first two args the same
let llvm_name =
&format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width);
let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
// llvm expects shift to be the same type as the values, but rust
// always uses `u32`.
let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
self.call_intrinsic(llvm_name, &[val, val, raw_shift])
self.call_intrinsic(&llvm_name, &[llty], &[val, val, raw_shift])
}
sym::saturating_add | sym::saturating_sub => {
let is_add = name == sym::saturating_add;
let lhs = args[0].immediate();
let rhs = args[1].immediate();
let llvm_name = &format!(
"llvm.{}{}.sat.i{}",
let llvm_name = format!(
"llvm.{}{}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" },
width
);
self.call_intrinsic(llvm_name, &[lhs, rhs])
self.call_intrinsic(&llvm_name, &[llty], &[lhs, rhs])
}
_ => bug!(),
}
@@ -484,11 +466,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
self.icmp(IntPredicate::IntEQ, a_val, b_val)
} else {
let n = self.const_usize(layout.size().bytes());
let cmp = self.call_intrinsic("memcmp", &[a, b, n]);
match self.cx.sess().target.arch.as_ref() {
"avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)),
_ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)),
}
let cmp = self.call_intrinsic("memcmp", &[], &[a, b, n]);
self.icmp(IntPredicate::IntEQ, cmp, self.const_int(self.type_int(), 0))
}
}
@@ -496,6 +475,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
// Here we assume that the `memcmp` provided by the target is a NOP for size 0.
let cmp = self.call_intrinsic(
"memcmp",
&[],
&[args[0].immediate(), args[1].immediate(), args[2].immediate()],
);
// Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`.
@@ -619,18 +599,22 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
}
fn abort(&mut self) {
self.call_intrinsic("llvm.trap", &[]);
self.call_intrinsic("llvm.trap", &[], &[]);
}
fn assume(&mut self, val: Self::Value) {
if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
self.call_intrinsic("llvm.assume", &[val]);
self.call_intrinsic("llvm.assume", &[], &[val]);
}
}
fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value {
if self.cx.sess().opts.optimize != rustc_session::config::OptLevel::No {
self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)])
self.call_intrinsic(
"llvm.expect",
&[self.type_i1()],
&[cond, self.const_bool(expected)],
)
} else {
cond
}
@@ -644,17 +628,20 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
) -> Self::Value {
let typeid = self.get_metadata_value(typeid);
let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
let type_checked_load =
self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]);
let type_checked_load = self.call_intrinsic(
"llvm.type.checked.load",
&[],
&[llvtable, vtable_byte_offset, typeid],
);
self.extract_value(type_checked_load, 0)
}
fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
self.call_intrinsic("llvm.va_start", &[va_list])
self.call_intrinsic("llvm.va_start", &[self.type_ptr()], &[va_list])
}
fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value {
self.call_intrinsic("llvm.va_end", &[va_list])
self.call_intrinsic("llvm.va_end", &[self.type_ptr()], &[va_list])
}
}
@@ -893,8 +880,8 @@ fn codegen_wasm_try<'ll, 'tcx>(
let null = bx.const_null(bx.type_ptr());
let funclet = bx.catch_pad(cs, &[null]);
let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]);
let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]);
let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[], &[funclet.cleanuppad()]);
let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]);
let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void());
bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None);
@@ -1031,7 +1018,7 @@ fn codegen_emcc_try<'ll, 'tcx>(
let selector = bx.extract_value(vals, 1);
// Check if the typeid we got is the one for a Rust panic.
let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]);
let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.type_ptr()], &[tydesc]);
let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid);
let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool());
@@ -1522,56 +1509,37 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
}};
}
let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() {
let elem_ty = bx.cx.type_float_from_ty(*f);
match f.bit_width() {
16 => ("f16", elem_ty),
32 => ("f32", elem_ty),
64 => ("f64", elem_ty),
128 => ("f128", elem_ty),
_ => return_error!(InvalidMonomorphization::FloatingPointVector {
span,
name,
f_ty: *f,
in_ty,
}),
}
let elem_ty = if let ty::Float(f) = in_elem.kind() {
bx.cx.type_float_from_ty(*f)
} else {
return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty });
};
let vec_ty = bx.type_vector(elem_ty, in_len);
let (intr_name, fn_ty) = match name {
sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_relaxed_fma => ("fmuladd", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)),
sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)),
sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)),
let intr_name = match name {
sym::simd_ceil => "llvm.ceil",
sym::simd_fabs => "llvm.fabs",
sym::simd_fcos => "llvm.cos",
sym::simd_fexp2 => "llvm.exp2",
sym::simd_fexp => "llvm.exp",
sym::simd_flog10 => "llvm.log10",
sym::simd_flog2 => "llvm.log2",
sym::simd_flog => "llvm.log",
sym::simd_floor => "llvm.floor",
sym::simd_fma => "llvm.fma",
sym::simd_relaxed_fma => "llvm.fmuladd",
sym::simd_fsin => "llvm.sin",
sym::simd_fsqrt => "llvm.sqrt",
sym::simd_round => "llvm.round",
sym::simd_trunc => "llvm.trunc",
_ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
};
let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}");
let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty);
let c = bx.call(
fn_ty,
None,
None,
f,
Ok(bx.call_intrinsic(
intr_name,
&[vec_ty],
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
None,
None,
);
Ok(c)
))
}
if std::matches!(
@@ -1595,29 +1563,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args);
}
// FIXME: use:
// https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182
// https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81
fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String {
match *elem_ty.kind() {
ty::Int(v) => format!(
"v{}i{}",
vec_len,
// Normalize to prevent crash if v: IntTy::Isize
v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
),
ty::Uint(v) => format!(
"v{}i{}",
vec_len,
// Normalize to prevent crash if v: UIntTy::Usize
v.normalize(bx.target_spec().pointer_width).bit_width().unwrap()
),
ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()),
ty::RawPtr(_, _) => format!("v{}p0", vec_len),
_ => unreachable!(),
}
}
fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type {
let elem_ty = match *elem_ty.kind() {
ty::Int(v) => cx.type_int_from_ty(v),
@@ -1698,38 +1643,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
);
// Alignment of T, must be a constant integer value:
let alignment_ty = bx.type_i32();
let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
// Truncate the mask vector to a vector of i1s:
let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
let mask_ty = bx.type_vector(bx.type_i1(), in_len);
// Type of the vector of pointers:
let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
let llvm_intrinsic =
format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
let fn_ty = bx.type_func(
&[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty],
llvm_elem_vec_ty,
);
let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(
fn_ty,
None,
None,
f,
return Ok(bx.call_intrinsic(
"llvm.masked.gather",
&[llvm_elem_vec_ty, llvm_pointer_vec_ty],
&[args[1].immediate(), alignment, mask, args[0].immediate()],
None,
None,
);
return Ok(v);
));
}
if name == sym::simd_masked_load {
@@ -1795,32 +1724,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
);
let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
// Alignment of T, must be a constant integer value:
let alignment_ty = bx.type_i32();
let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
let llvm_pointer = bx.type_ptr();
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0");
let fn_ty = bx
.type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty);
let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(
fn_ty,
None,
None,
f,
return Ok(bx.call_intrinsic(
"llvm.masked.load",
&[llvm_elem_vec_ty, llvm_pointer],
&[args[1].immediate(), alignment, mask, args[2].immediate()],
None,
None,
);
return Ok(v);
));
}
if name == sym::simd_masked_store {
@@ -1880,33 +1797,20 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
);
let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
// Alignment of T, must be a constant integer value:
let alignment_ty = bx.type_i32();
let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
let ret_t = bx.type_void();
let llvm_pointer = bx.type_ptr();
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(
fn_ty,
None,
None,
f,
return Ok(bx.call_intrinsic(
"llvm.masked.store",
&[llvm_elem_vec_ty, llvm_pointer],
&[args[2].immediate(), args[1].immediate(), alignment, mask],
None,
None,
);
return Ok(v);
));
}
if name == sym::simd_scatter {
@@ -1971,38 +1875,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
);
// Alignment of T, must be a constant integer value:
let alignment_ty = bx.type_i32();
let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32);
// Truncate the mask vector to a vector of i1s:
let mask = vector_mask_to_bitmask(bx, args[2].immediate(), mask_elem_bitwidth, in_len);
let mask_ty = bx.type_vector(bx.type_i1(), in_len);
let ret_t = bx.type_void();
// Type of the vector of pointers:
let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len);
let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len);
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len);
let llvm_intrinsic =
format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}");
let fn_ty =
bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t);
let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(
fn_ty,
None,
None,
f,
return Ok(bx.call_intrinsic(
"llvm.masked.scatter",
&[llvm_elem_vec_ty, llvm_pointer_vec_ty],
&[args[0].immediate(), args[1].immediate(), alignment, mask],
None,
None,
);
return Ok(v);
));
}
macro_rules! arith_red {
@@ -2431,40 +2319,31 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
},
in_len as u64,
);
let intrinsic_name = match name {
sym::simd_bswap => "bswap",
sym::simd_bitreverse => "bitreverse",
sym::simd_ctlz => "ctlz",
sym::simd_ctpop => "ctpop",
sym::simd_cttz => "cttz",
let llvm_intrinsic = match name {
sym::simd_bswap => "llvm.bswap",
sym::simd_bitreverse => "llvm.bitreverse",
sym::simd_ctlz => "llvm.ctlz",
sym::simd_ctpop => "llvm.ctpop",
sym::simd_cttz => "llvm.cttz",
_ => unreachable!(),
};
let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
return match name {
// byte swap is no-op for i8/u8
sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
sym::simd_ctlz | sym::simd_cttz => {
// for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison`
let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
Ok(bx.call(
fn_ty,
None,
None,
f,
Ok(bx.call_intrinsic(
llvm_intrinsic,
&[vec_ty],
&[args[0].immediate(), dont_poison_on_zero],
None,
None,
))
}
sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
// simple unary argument cases
let fn_ty = bx.type_func(&[vec_ty], vec_ty);
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[args[0].immediate()]))
}
_ => unreachable!(),
};
@@ -2495,10 +2374,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
let lhs = args[0].immediate();
let rhs = args[1].immediate();
let is_add = name == sym::simd_saturating_add;
let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _;
let (signed, elem_width, elem_ty) = match *in_elem.kind() {
ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)),
ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)),
let (signed, elem_ty) = match *in_elem.kind() {
ty::Int(i) => (true, bx.cx.type_int_from_ty(i)),
ty::Uint(i) => (false, bx.cx.type_uint_from_ty(i)),
_ => {
return_error!(InvalidMonomorphization::ExpectedVectorElementType {
span,
@@ -2508,19 +2386,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
});
}
};
let llvm_intrinsic = &format!(
"llvm.{}{}.sat.v{}i{}",
let llvm_intrinsic = format!(
"llvm.{}{}.sat",
if signed { 's' } else { 'u' },
if is_add { "add" } else { "sub" },
in_len,
elem_width
);
let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64);
let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty);
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None, None);
return Ok(v);
return Ok(bx.call_intrinsic(&llvm_intrinsic, &[vec_ty], &[lhs, rhs]));
}
span_bug!(span, "unknown SIMD intrinsic");

12
compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View File

@@ -15,6 +15,7 @@
use std::fmt::Debug;
use std::marker::PhantomData;
use std::num::NonZero;
use std::ptr;
use bitflags::bitflags;
@@ -1195,6 +1196,17 @@ unsafe extern "C" {
// Operations on functions
pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint);
// Operations about llvm intrinsics
pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint;
pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero<c_uint>) -> Bool;
pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>(
Mod: &'a Module,
ID: NonZero<c_uint>,
ParamTypes: *const &'a Type,
ParamCount: size_t,
NameLength: *mut size_t,
) -> *mut c_char;
// Operations on parameters
pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
pub(crate) safe fn LLVMCountParams(Fn: &Value) -> c_uint;

45
compiler/rustc_codegen_llvm/src/llvm/mod.rs
View File

@@ -1,9 +1,10 @@
#![allow(non_snake_case)]
use std::ffi::{CStr, CString};
use std::ptr;
use std::num::NonZero;
use std::str::FromStr;
use std::string::FromUtf8Error;
use std::{ptr, slice};
use libc::c_uint;
use rustc_abi::{Align, Size, WrappingRange};
@@ -327,6 +328,48 @@ pub(crate) fn get_value_name(value: &Value) -> &[u8] {
}
}
#[derive(Debug, Copy, Clone)]
pub(crate) struct Intrinsic {
id: NonZero<c_uint>,
}
impl Intrinsic {
pub(crate) fn lookup(name: &[u8]) -> Option<Self> {
let id = unsafe { LLVMLookupIntrinsicID(name.as_c_char_ptr(), name.len()) };
NonZero::new(id).map(|id| Self { id })
}
pub(crate) fn is_overloaded(self) -> bool {
unsafe { LLVMIntrinsicIsOverloaded(self.id) == True }
}
pub(crate) fn overloaded_name<'ll>(
self,
llmod: &'ll Module,
type_params: &[&'ll Type],
) -> String {
let mut len = 0;
let ptr = unsafe {
LLVMIntrinsicCopyOverloadedName2(
llmod,
self.id,
type_params.as_ptr(),
type_params.len(),
&mut len,
)
};
let slice = unsafe { slice::from_raw_parts_mut(ptr.cast(), len) };
let copied = str::from_utf8(slice).expect("Non-UTF8 intrinsic name").to_string();
unsafe {
libc::free(ptr.cast());
}
copied
}
}
/// Safe wrapper for `LLVMSetValueName2` from a byte slice
pub(crate) fn set_value_name(value: &Value, name: &[u8]) {
unsafe {

9
compiler/rustc_codegen_llvm/src/type_.rs
View File

@@ -1,4 +1,5 @@
use std::borrow::Borrow;
use std::hash::{Hash, Hasher};
use std::{fmt, ptr};
use libc::{c_char, c_uint};
@@ -25,6 +26,14 @@ impl PartialEq for Type {
}
}
impl Eq for Type {}
impl Hash for Type {
fn hash<H: Hasher>(&self, state: &mut H) {
ptr::hash(self, state);
}
}
impl fmt::Debug for Type {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(