rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge commit '35d9c6bf256968e1b40e0d554607928bdf9cebea' into sync_cg_clif-2022-02-23

Browse Source
This commit is contained in:
bjorn3
2022-02-23 11:49:34 +01:00
parent c651ba8a54 35d9c6bf25
commit f596dce542
24 changed files with 843 additions and 809 deletions
Download Patch File Download Diff File Expand all files Collapse all files

445
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View File

@@ -1,71 +1,116 @@
//! Codegen `extern "platform-intrinsic"` intrinsics.
use rustc_middle::ty::subst::SubstsRef;
use rustc_span::Symbol;
use super::*;
use crate::prelude::*;
fn report_simd_type_validation_error(
fx: &mut FunctionCx<'_, '_, '_>,
intrinsic: Symbol,
span: Span,
ty: Ty<'_>,
) {
fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
// Prevent verifier error
crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
}
pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
instance: Instance<'tcx>,
intrinsic: Symbol,
_substs: SubstsRef<'tcx>,
args: &[mir::Operand<'tcx>],
ret: CPlace<'tcx>,
span: Span,
) {
let def_id = instance.def_id();
let substs = instance.substs;
let intrinsic = fx.tcx.item_name(def_id);
intrinsic_match! {
fx, intrinsic, substs, args,
fx, intrinsic, args,
_ => {
fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
};
simd_cast, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
let from_signed = type_sign(lane_layout.ty);
let to_signed = type_sign(ret_lane_layout.ty);
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
CValue::by_val(ret_lane, ret_lane_layout)
let from_signed = type_sign(lane_ty);
let to_signed = type_sign(ret_lane_ty);
clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
});
};
simd_eq, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_cmp!(fx, Equal|Equal(x, y) -> ret);
};
simd_ne, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret);
};
simd_lt, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
};
simd_le, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
};
simd_gt, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
};
simd_ge, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_cmp!(
fx,
UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
(x, y) -> ret
);
simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) {
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
}
// FIXME use vector instructions when possible
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
let res_lane = match (lane_ty.kind(), intrinsic) {
(ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
(ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
(ty::Uint(_), sym::simd_lt) => {
fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
}
(ty::Uint(_), sym::simd_le) => {
fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
}
(ty::Uint(_), sym::simd_gt) => {
fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
}
(ty::Uint(_), sym::simd_ge) => {
fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
}
(ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
(ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
(ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
(ty::Int(_), sym::simd_le) => {
fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
}
(ty::Int(_), sym::simd_gt) => {
fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
}
(ty::Int(_), sym::simd_ge) => {
fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
}
(ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
(ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
(ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
(ty::Float(_), sym::simd_le) => {
fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
}
(ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
(ty::Float(_), sym::simd_ge) => {
fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
}
_ => unreachable!(),
};
let ty = fx.clif_type(res_lane_ty).unwrap();
let res_lane = fx.bcx.ins().bint(ty, res_lane);
fx.bcx.ins().ineg(res_lane)
});
};
// simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
_ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
}
// If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
// If there is no suffix, use the index array length.
@@ -167,7 +212,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_extract, (c v, o idx) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
idx_const
} else {
@@ -195,95 +244,82 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_neg, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
let ret_lane = match lane_layout.ty.kind() {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
match lane_ty.kind() {
ty::Int(_) => fx.bcx.ins().ineg(lane),
ty::Float(_) => fx.bcx.ins().fneg(lane),
_ => unreachable!(),
};
CValue::by_val(ret_lane, ret_lane_layout)
}
});
};
simd_fabs, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
let ret_lane = fx.bcx.ins().fabs(lane);
CValue::by_val(ret_lane, ret_lane_layout)
});
};
simd_add | simd_sub | simd_mul | simd_div | simd_rem
| simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
}
simd_fsqrt, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
let ret_lane = fx.bcx.ins().sqrt(lane);
CValue::by_val(ret_lane, ret_lane_layout)
});
};
// FIXME use vector instructions when possible
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
lane_ty.kind(),
intrinsic,
) {
(ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
(ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
(ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
(ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
(ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
simd_add, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
};
simd_sub, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
};
simd_mul, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
};
simd_div, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
};
simd_rem, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
let res_lane = match lane_layout.ty.kind() {
ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane),
ty::Float(FloatTy::F32) => fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x_lane, y_lane],
)[0],
ty::Float(FloatTy::F64) => fx.lib_call(
"fmod",
vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
vec![AbiParam::new(types::F64)],
&[x_lane, y_lane],
)[0],
_ => unreachable!("{:?}", lane_layout.ty),
};
CValue::by_val(res_lane, ret_lane_layout)
(ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
(ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
(ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
(ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
(ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
(ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
(ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
(ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
(ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
(ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x_lane, y_lane],
)[0],
(ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
"fmod",
vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
vec![AbiParam::new(types::F64)],
&[x_lane, y_lane],
)[0],
(ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
(ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
(ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
(ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
(ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
(ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
(ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
(ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
_ => unreachable!(),
});
};
simd_shl, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_binop!(fx, ishl(x, y) -> ret);
};
simd_shr, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
};
simd_and, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_binop!(fx, band(x, y) -> ret);
};
simd_or, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_binop!(fx, bor(x, y) -> ret);
};
simd_xor, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_int_binop!(fx, bxor(x, y) -> ret);
};
simd_fma, (c a, c b, c c) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
assert_eq!(a.layout(), b.layout());
assert_eq!(a.layout(), c.layout());
let layout = a.layout();
@@ -305,19 +341,34 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
}
};
simd_fmin, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_flt_binop!(fx, fmin(x, y) -> ret);
};
simd_fmax, (c x, c y) {
validate_simd_type!(fx, intrinsic, span, x.layout().ty);
simd_flt_binop!(fx, fmax(x, y) -> ret);
simd_fmin | simd_fmax, (c x, c y) {
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
}
// FIXME use vector instructions when possible
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
match lane_ty.kind() {
ty::Float(_) => {},
_ => unreachable!("{:?}", lane_ty),
}
match intrinsic {
sym::simd_fmin => fx.bcx.ins().fmin(x_lane, y_lane),
sym::simd_fmax => fx.bcx.ins().fmax(x_lane, y_lane),
_ => unreachable!(),
}
});
};
simd_round, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
let res_lane = match lane_layout.ty.kind() {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
match lane_ty.kind() {
ty::Float(FloatTy::F32) => fx.lib_call(
"roundf",
vec![AbiParam::new(types::F32)],
@@ -330,37 +381,41 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
vec![AbiParam::new(types::F64)],
&[lane],
)[0],
_ => unreachable!("{:?}", lane_layout.ty),
};
CValue::by_val(res_lane, ret_lane_layout)
_ => unreachable!("{:?}", lane_ty),
}
});
};
simd_ceil, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
let ret_lane = fx.bcx.ins().ceil(lane);
CValue::by_val(ret_lane, ret_lane_layout)
});
};
simd_floor, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
let ret_lane = fx.bcx.ins().floor(lane);
CValue::by_val(ret_lane, ret_lane_layout)
});
};
simd_trunc, (c a) {
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| {
let ret_lane = fx.bcx.ins().trunc(lane);
CValue::by_val(ret_lane, ret_lane_layout)
simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
match lane_ty.kind() {
ty::Float(_) => {},
_ => unreachable!("{:?}", lane_ty),
}
match intrinsic {
sym::simd_fabs => fx.bcx.ins().fabs(lane),
sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
sym::simd_ceil => fx.bcx.ins().ceil(lane),
sym::simd_floor => fx.bcx.ins().floor(lane),
sym::simd_trunc => fx.bcx.ins().trunc(lane),
_ => unreachable!(),
}
});
};
simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| {
if lane_layout.ty.is_floating_point() {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
if lane_ty.is_floating_point() {
fx.bcx.ins().fadd(a, b)
} else {
fx.bcx.ins().iadd(a, b)
@@ -369,9 +424,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| {
if lane_layout.ty.is_floating_point() {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
if lane_ty.is_floating_point() {
fx.bcx.ins().fmul(a, b)
} else {
fx.bcx.ins().imul(a, b)
@@ -380,34 +439,58 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_reduce_all, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b));
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
};
simd_reduce_any, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b));
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
};
simd_reduce_and, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().band(a, b));
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
};
simd_reduce_or, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bor(a, b));
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
};
simd_reduce_xor, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bxor(a, b));
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
};
simd_reduce_min, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
let lt = match layout.ty.kind() {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
let lt = match ty.kind() {
ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
@@ -418,9 +501,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_reduce_max, (c v) {
validate_simd_type!(fx, intrinsic, span, v.layout().ty);
simd_reduce(fx, v, None, ret, |fx, layout, a, b| {
let gt = match layout.ty.kind() {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
let gt = match ty.kind() {
ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
@@ -431,8 +518,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
simd_select, (c m, c a, c b) {
validate_simd_type!(fx, intrinsic, span, m.layout().ty);
validate_simd_type!(fx, intrinsic, span, a.layout().ty);
if !m.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
return;
}
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
assert_eq!(a.layout(), b.layout());
let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);