intrinsics: use const generic to set atomic ordering

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -875,7 +875,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let ptr = ptr.load_scalar(fx);
 
             let ty = generic_args.type_at(0);
-            let _ord = generic_args.const_at(1).to_value(); // FIXME: forward this to cranelift once they support that
             match ty.kind() {
                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
                     // FIXME implement 128bit atomics
@@ -906,7 +905,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(ty));
             ret.write_cvalue(fx, val);
         }
-        _ if intrinsic.as_str().starts_with("atomic_store") => {
+        sym::atomic_store => {
             intrinsic_args!(fx, args => (ptr, val); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -939,7 +938,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
             fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
         }
-        _ if intrinsic.as_str().starts_with("atomic_xchg") => {
+        sym::atomic_xchg => {
             intrinsic_args!(fx, args => (ptr, new); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -960,8 +959,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_cxchg") => {
+        sym::atomic_cxchg | sym::atomic_cxchgweak => {
-            // both atomic_cxchg_* and atomic_cxchgweak_*
             intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -984,7 +982,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, ret_val)
         }
 
-        _ if intrinsic.as_str().starts_with("atomic_xadd") => {
+        sym::atomic_xadd => {
             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1006,7 +1004,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_xsub") => {
+        sym::atomic_xsub => {
             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1028,7 +1026,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_and") => {
+        sym::atomic_and => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1049,7 +1047,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_or") => {
+        sym::atomic_or => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1070,7 +1068,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_xor") => {
+        sym::atomic_xor => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1091,7 +1089,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_nand") => {
+        sym::atomic_nand => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1112,7 +1110,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_max") => {
+        sym::atomic_max => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1133,7 +1131,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_umax") => {
+        sym::atomic_umax => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1154,7 +1152,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_min") => {
+        sym::atomic_min => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1175,7 +1173,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_umin") => {
+        sym::atomic_umin => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 

compiler/rustc_codegen_ssa/messages.ftl

@@ -8,8 +8,6 @@ codegen_ssa_aix_strip_not_used = using host's `strip` binary to cross-compile to
 
 codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
 
-codegen_ssa_atomic_compare_exchange = Atomic compare-exchange intrinsic missing failure memory ordering
-
 codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto
 
 codegen_ssa_bare_instruction_set = `#[instruction_set]` requires an argument
@@ -206,8 +204,6 @@ codegen_ssa_missing_cpp_build_tool_component = or a necessary component may be m
 
 codegen_ssa_missing_features = add the missing features in a `target_feature` attribute
 
-codegen_ssa_missing_memory_ordering = Atomic intrinsic missing memory ordering
-
 codegen_ssa_missing_query_depgraph =
     found CGU-reuse attribute but `-Zquery-dep-graph` was not specified
 
@@ -374,10 +370,6 @@ codegen_ssa_unexpected_parameter_name = unexpected parameter name
 codegen_ssa_unknown_archive_kind =
     Don't know how to build archive of type: {$kind}
 
-codegen_ssa_unknown_atomic_operation = unknown atomic operation
-
-codegen_ssa_unknown_atomic_ordering = unknown ordering in atomic intrinsic
-
 codegen_ssa_unknown_reuse_kind = unknown cgu-reuse-kind `{$kind}` specified
 
 codegen_ssa_unsupported_instruction_set = target does not support `#[instruction_set]`

compiler/rustc_codegen_ssa/src/errors.rs

@@ -796,22 +796,6 @@ pub(crate) struct ShuffleIndicesEvaluation {
     pub span: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_missing_memory_ordering)]
-pub(crate) struct MissingMemoryOrdering;
-
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_unknown_atomic_ordering)]
-pub(crate) struct UnknownAtomicOrdering;
-
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_atomic_compare_exchange)]
-pub(crate) struct AtomicCompareExchange;
-
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_unknown_atomic_operation)]
-pub(crate) struct UnknownAtomicOperation;
-
 #[derive(Diagnostic)]
 pub enum InvalidMonomorphization<'tcx> {
     #[diag(codegen_ssa_invalid_monomorphization_basic_integer_type, code = E0511)]

compiler/rustc_codegen_ssa/src/mir/intrinsic.rs

@@ -8,9 +8,10 @@ use rustc_span::sym;
 use super::FunctionCx;
 use super::operand::OperandRef;
 use super::place::PlaceRef;
+use crate::common::{AtomicRmwBinOp, SynchronizationScope};
 use crate::errors::InvalidMonomorphization;
 use crate::traits::*;
-use crate::{MemFlags, errors, meth, size_of_val};
+use crate::{MemFlags, meth, size_of_val};
 
 fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx: &mut Bx,
@@ -62,7 +63,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let span = source_info.span;
 
         let name = bx.tcx().item_name(instance.def_id());
-        let name_str = name.as_str();
         let fn_args = instance.args;
 
         // If we're swapping something that's *not* an `OperandValue::Ref`,
@@ -89,14 +89,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
         }
 
-        let ret_llval = |bx: &mut Bx, llval| {
+        let invalid_monomorphization_int_type = |ty| {
-            if result.layout.ty.is_bool() {
+            bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
-                let val = bx.from_immediate(llval);
+        };
-                bx.store_to_place(val, result.val);
+
-            } else if !result.layout.ty.is_unit() {
+        let parse_atomic_ordering = |ord: ty::Value<'tcx>| {
-                bx.store_to_place(llval, result.val);
+            let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
-            }
+            discr.to_atomic_ordering()
-            Ok(())
         };
 
         let llval = match name {
@@ -336,183 +335,144 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            // This requires that atomic intrinsics follow a specific naming pattern:
+            sym::atomic_load => {
-            // "atomic_<operation>[_<ordering>]"
+                let ty = fn_args.type_at(0);
-            name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
+                if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
-                use rustc_middle::ty::AtomicOrdering::*;
+                    invalid_monomorphization_int_type(ty);
-
+                    return Ok(());
-                use crate::common::{AtomicRmwBinOp, SynchronizationScope};
-
-                let invalid_monomorphization = |ty| {
-                    bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
-                        span,
-                        name,
-                        ty,
-                    });
-                };
-
-                let parse_const_generic_ordering = |ord: ty::Value<'tcx>| {
-                    let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
-                    discr.to_atomic_ordering()
-                };
-
-                // Some intrinsics have the ordering already converted to a const generic parameter, we handle those first.
-                match name {
-                    sym::atomic_load => {
-                        let ty = fn_args.type_at(0);
-                        let ordering = fn_args.const_at(1).to_value();
-                        if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                        let layout = bx.layout_of(ty);
-                        let source = args[0].immediate();
-                        let llval = bx.atomic_load(
-                            bx.backend_type(layout),
-                            source,
-                            parse_const_generic_ordering(ordering),
-                            layout.size,
-                        );
-
-                        return ret_llval(bx, llval);
-                    }
-
-                    // The rest falls back to below.
-                    _ => {}
                 }
-
+                let ordering = fn_args.const_at(1).to_value();
-                let Some((instruction, ordering)) = atomic.split_once('_') else {
+                let layout = bx.layout_of(ty);
-                    bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
+                let source = args[0].immediate();
-                };
+                bx.atomic_load(
-
+                    bx.backend_type(layout),
-                let parse_ordering = |bx: &Bx, s| match s {
+                    source,
-                    "relaxed" => Relaxed,
+                    parse_atomic_ordering(ordering),
-                    "acquire" => Acquire,
+                    layout.size,
-                    "release" => Release,
+                )
-                    "acqrel" => AcqRel,
+            }
-                    "seqcst" => SeqCst,
+            sym::atomic_store => {
-                    _ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
+                let ty = fn_args.type_at(0);
-                };
+                if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
-
+                    invalid_monomorphization_int_type(ty);
-                match instruction {
+                    return Ok(());
-                    "cxchg" | "cxchgweak" => {
-                        let Some((success, failure)) = ordering.split_once('_') else {
-                            bx.sess().dcx().emit_fatal(errors::AtomicCompareExchange);
-                        };
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let weak = instruction == "cxchgweak";
-                            let dst = args[0].immediate();
-                            let cmp = args[1].immediate();
-                            let src = args[2].immediate();
-                            let (val, success) = bx.atomic_cmpxchg(
-                                dst,
-                                cmp,
-                                src,
-                                parse_ordering(bx, success),
-                                parse_ordering(bx, failure),
-                                weak,
-                            );
-                            let val = bx.from_immediate(val);
-                            let success = bx.from_immediate(success);
-
-                            let dest = result.project_field(bx, 0);
-                            bx.store_to_place(val, dest.val);
-                            let dest = result.project_field(bx, 1);
-                            bx.store_to_place(success, dest.val);
-                        } else {
-                            invalid_monomorphization(ty);
-                        }
-                        return Ok(());
-                    }
-
-                    "store" => {
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let size = bx.layout_of(ty).size;
-                            let val = args[1].immediate();
-                            let ptr = args[0].immediate();
-                            bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
-                        } else {
-                            invalid_monomorphization(ty);
-                        }
-                        return Ok(());
-                    }
-
-                    "fence" => {
-                        bx.atomic_fence(
-                            parse_ordering(bx, ordering),
-                            SynchronizationScope::CrossThread,
-                        );
-                        return Ok(());
-                    }
-
-                    "singlethreadfence" => {
-                        bx.atomic_fence(
-                            parse_ordering(bx, ordering),
-                            SynchronizationScope::SingleThread,
-                        );
-                        return Ok(());
-                    }
-
-                    // These are all AtomicRMW ops
-                    "max" | "min" => {
-                        let atom_op = if instruction == "max" {
-                            AtomicRmwBinOp::AtomicMax
-                        } else {
-                            AtomicRmwBinOp::AtomicMin
-                        };
-
-                        let ty = fn_args.type_at(0);
-                        if matches!(ty.kind(), ty::Int(_)) {
-                            let ptr = args[0].immediate();
-                            let val = args[1].immediate();
-                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
-                    "umax" | "umin" => {
-                        let atom_op = if instruction == "umax" {
-                            AtomicRmwBinOp::AtomicUMax
-                        } else {
-                            AtomicRmwBinOp::AtomicUMin
-                        };
-
-                        let ty = fn_args.type_at(0);
-                        if matches!(ty.kind(), ty::Uint(_)) {
-                            let ptr = args[0].immediate();
-                            let val = args[1].immediate();
-                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
-                    op => {
-                        let atom_op = match op {
-                            "xchg" => AtomicRmwBinOp::AtomicXchg,
-                            "xadd" => AtomicRmwBinOp::AtomicAdd,
-                            "xsub" => AtomicRmwBinOp::AtomicSub,
-                            "and" => AtomicRmwBinOp::AtomicAnd,
-                            "nand" => AtomicRmwBinOp::AtomicNand,
-                            "or" => AtomicRmwBinOp::AtomicOr,
-                            "xor" => AtomicRmwBinOp::AtomicXor,
-                            _ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOperation),
-                        };
-
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let ptr = args[0].immediate();
-                            let val = args[1].immediate();
-                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
                 }
+                let ordering = fn_args.const_at(1).to_value();
+                let size = bx.layout_of(ty).size;
+                let val = args[1].immediate();
+                let ptr = args[0].immediate();
+                bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size);
+                return Ok(());
+            }
+            sym::atomic_cxchg | sym::atomic_cxchgweak => {
+                let ty = fn_args.type_at(0);
+                if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+                let succ_ordering = fn_args.const_at(1).to_value();
+                let fail_ordering = fn_args.const_at(2).to_value();
+                let weak = name == sym::atomic_cxchgweak;
+                let dst = args[0].immediate();
+                let cmp = args[1].immediate();
+                let src = args[2].immediate();
+                let (val, success) = bx.atomic_cmpxchg(
+                    dst,
+                    cmp,
+                    src,
+                    parse_atomic_ordering(succ_ordering),
+                    parse_atomic_ordering(fail_ordering),
+                    weak,
+                );
+                let val = bx.from_immediate(val);
+                let success = bx.from_immediate(success);
+
+                let dest = result.project_field(bx, 0);
+                bx.store_to_place(val, dest.val);
+                let dest = result.project_field(bx, 1);
+                bx.store_to_place(success, dest.val);
+
+                return Ok(());
+            }
+            // These are all AtomicRMW ops
+            sym::atomic_max | sym::atomic_min => {
+                let atom_op = if name == sym::atomic_max {
+                    AtomicRmwBinOp::AtomicMax
+                } else {
+                    AtomicRmwBinOp::AtomicMin
+                };
+
+                let ty = fn_args.type_at(0);
+                if matches!(ty.kind(), ty::Int(_)) {
+                    let ordering = fn_args.const_at(1).to_value();
+                    let ptr = args[0].immediate();
+                    let val = args[1].immediate();
+                    bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
+                } else {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+            }
+            sym::atomic_umax | sym::atomic_umin => {
+                let atom_op = if name == sym::atomic_umax {
+                    AtomicRmwBinOp::AtomicUMax
+                } else {
+                    AtomicRmwBinOp::AtomicUMin
+                };
+
+                let ty = fn_args.type_at(0);
+                if matches!(ty.kind(), ty::Uint(_)) {
+                    let ordering = fn_args.const_at(1).to_value();
+                    let ptr = args[0].immediate();
+                    let val = args[1].immediate();
+                    bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
+                } else {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+            }
+            sym::atomic_xchg
+            | sym::atomic_xadd
+            | sym::atomic_xsub
+            | sym::atomic_and
+            | sym::atomic_nand
+            | sym::atomic_or
+            | sym::atomic_xor => {
+                let atom_op = match name {
+                    sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg,
+                    sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd,
+                    sym::atomic_xsub => AtomicRmwBinOp::AtomicSub,
+                    sym::atomic_and => AtomicRmwBinOp::AtomicAnd,
+                    sym::atomic_nand => AtomicRmwBinOp::AtomicNand,
+                    sym::atomic_or => AtomicRmwBinOp::AtomicOr,
+                    sym::atomic_xor => AtomicRmwBinOp::AtomicXor,
+                    _ => unreachable!(),
+                };
+
+                let ty = fn_args.type_at(0);
+                if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
+                    let ordering = fn_args.const_at(1).to_value();
+                    let ptr = args[0].immediate();
+                    let val = args[1].immediate();
+                    bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
+                } else {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+            }
+            sym::atomic_fence => {
+                let ordering = fn_args.const_at(0).to_value();
+                bx.atomic_fence(parse_atomic_ordering(ordering), SynchronizationScope::CrossThread);
+                return Ok(());
+            }
+
+            sym::atomic_singlethreadfence => {
+                let ordering = fn_args.const_at(0).to_value();
+                bx.atomic_fence(
+                    parse_atomic_ordering(ordering),
+                    SynchronizationScope::SingleThread,
+                );
+                return Ok(());
             }
 
             sym::nontemporal_store => {
@@ -556,7 +516,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
         };
 
-        ret_llval(bx, llval)
+        if result.layout.ty.is_bool() {
+            let val = bx.from_immediate(llval);
+            bx.store_to_place(val, result.val);
+        } else if !result.layout.ty.is_unit() {
+            bx.store_to_place(llval, result.val);
+        }
+        Ok(())
     }
 }
 

compiler/rustc_error_codes/src/error_codes/E0092.md

@@ -1,8 +1,10 @@
+#### Note: this error code is no longer emitted by the compiler.
+
 An undefined atomic operation function was declared.
 
 Erroneous code example:
 
-```compile_fail,E0092
+```ignore (no longer emitted)
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
@@ -12,13 +14,4 @@ unsafe fn atomic_foo(); // error: unrecognized atomic operation
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
-functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
+functions are defined in `library/core/src/intrinsics` in the Rust source code.
-`library/core/src/intrinsics.rs` in the Rust source code. Example:
-
-```
-#![feature(intrinsics)]
-#![allow(internal_features)]
-
-#[rustc_intrinsic]
-unsafe fn atomic_fence_seqcst(); // ok!
-```

compiler/rustc_error_codes/src/error_codes/E0093.md

@@ -17,19 +17,4 @@ fn main() {
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
-functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
+functions are defined in `library/core/src/intrinsics` in the Rust source code.
-`library/core/src/intrinsics.rs` in the Rust source code. Example:
-
-```
-#![feature(intrinsics)]
-#![allow(internal_features)]
-
-#[rustc_intrinsic]
-unsafe fn atomic_fence_seqcst(); // ok!
-
-fn main() {
-    unsafe {
-        atomic_fence_seqcst();
-    }
-}
-```

compiler/rustc_error_codes/src/error_codes/E0622.md

@@ -4,7 +4,7 @@ An intrinsic was declared without being a function.
 
 Erroneous code example:
 
-```no_run
+```ignore (no longer emitted)
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
@@ -21,7 +21,7 @@ An intrinsic is a function available for use in a given programming language
 whose implementation is handled specially by the compiler. In order to fix this
 error, just declare a function. Example:
 
-```no_run
+```ignore (no longer emitted)
 #![feature(intrinsics)]
 #![allow(internal_features)]
 

compiler/rustc_hir_analysis/messages.ftl

@@ -565,10 +565,6 @@ hir_analysis_unconstrained_generic_parameter = the {$param_def_kind} `{$param_na
 hir_analysis_unconstrained_opaque_type = unconstrained opaque type
     .note = `{$name}` must be used in combination with a concrete type within the same {$what}
 
-hir_analysis_unrecognized_atomic_operation =
-    unrecognized atomic operation function: `{$op}`
-    .label = unrecognized atomic operation
-
 hir_analysis_unrecognized_intrinsic_function =
     unrecognized intrinsic function: `{$name}`
     .label = unrecognized intrinsic

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -9,10 +9,7 @@ use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, Symbol, sym};
 
 use crate::check::check_function_signature;
-use crate::errors::{
+use crate::errors::{UnrecognizedIntrinsicFunction, WrongNumberOfGenericArgumentsToIntrinsic};
-    UnrecognizedAtomicOperation, UnrecognizedIntrinsicFunction,
-    WrongNumberOfGenericArgumentsToIntrinsic,
-};
 
 fn equate_intrinsic_type<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -172,7 +169,6 @@ pub(crate) fn check_intrinsic_type(
             Ty::new_error_with_message(tcx, span, "expected param")
         }
     };
-    let name_str = intrinsic_name.as_str();
 
     let bound_vars = tcx.mk_bound_variable_kinds(&[
         ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon),
@@ -198,32 +194,9 @@ pub(crate) fn check_intrinsic_type(
         (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty)
     };
 
-    let (n_tps, n_lts, n_cts, inputs, output, safety) = if name_str.starts_with("atomic_") {
+    let (n_tps, n_lts, n_cts, inputs, output, safety) = if intrinsic_name
-        let split: Vec<&str> = name_str.split('_').collect();
+        == sym::contract_check_ensures
-        assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
+    {
-
-        // Each atomic op has variants with different suffixes (`_seq_cst`, `_acquire`, etc.). Use
-        // string ops to strip the suffixes, because the variants all get the same treatment here.
-        let (n_tps, n_cts, inputs, output) = match split[1] {
-            "cxchg" | "cxchgweak" => (
-                1,
-                0,
-                vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
-                Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
-            ),
-            "load" => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
-            "store" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
-
-            "xchg" | "xadd" | "xsub" | "and" | "nand" | "or" | "xor" | "max" | "min" | "umax"
-            | "umin" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
-            "fence" | "singlethreadfence" => (0, 0, Vec::new(), tcx.types.unit),
-            op => {
-                tcx.dcx().emit_err(UnrecognizedAtomicOperation { span, op });
-                return;
-            }
-        };
-        (n_tps, 0, n_cts, inputs, output, hir::Safety::Unsafe)
-    } else if intrinsic_name == sym::contract_check_ensures {
         // contract_check_ensures::<Ret, C>(Ret, C) -> Ret
         // where C: for<'a> Fn(&'a Ret) -> bool,
         //
@@ -694,6 +667,30 @@ pub(crate) fn check_intrinsic_type(
             sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
             sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
 
+            sym::atomic_cxchg | sym::atomic_cxchgweak => (
+                1,
+                2,
+                vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
+                Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
+            ),
+            sym::atomic_load => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
+            sym::atomic_store => {
+                (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
+            }
+
+            sym::atomic_xchg
+            | sym::atomic_xadd
+            | sym::atomic_xsub
+            | sym::atomic_and
+            | sym::atomic_nand
+            | sym::atomic_or
+            | sym::atomic_xor
+            | sym::atomic_max
+            | sym::atomic_min
+            | sym::atomic_umax
+            | sym::atomic_umin => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
+            sym::atomic_fence | sym::atomic_singlethreadfence => (0, 1, Vec::new(), tcx.types.unit),
+
             other => {
                 tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span, name: other });
                 return;

compiler/rustc_hir_analysis/src/errors.rs

@@ -161,15 +161,6 @@ pub(crate) enum AssocItemNotFoundSugg<'a> {
     },
 }
 
-#[derive(Diagnostic)]
-#[diag(hir_analysis_unrecognized_atomic_operation, code = E0092)]
-pub(crate) struct UnrecognizedAtomicOperation<'a> {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-    pub op: &'a str,
-}
-
 #[derive(Diagnostic)]
 #[diag(hir_analysis_wrong_number_of_generic_arguments_to_intrinsic, code = E0094)]
 pub(crate) struct WrongNumberOfGenericArgumentsToIntrinsic<'a> {

compiler/rustc_span/src/symbol.rs

@@ -515,8 +515,24 @@ symbols! {
         async_iterator_poll_next,
         async_trait_bounds,
         atomic,
+        atomic_and,
+        atomic_cxchg,
+        atomic_cxchgweak,
+        atomic_fence,
         atomic_load,
+        atomic_max,
+        atomic_min,
         atomic_mod,
+        atomic_nand,
+        atomic_or,
+        atomic_singlethreadfence,
+        atomic_store,
+        atomic_umax,
+        atomic_umin,
+        atomic_xadd,
+        atomic_xchg,
+        atomic_xor,
+        atomic_xsub,
         atomics,
         att_syntax,
         attr,

library/core/src/sync/atomic.rs

@@ -245,6 +245,7 @@
 use self::Ordering::*;
 use crate::cell::UnsafeCell;
 use crate::hint::spin_loop;
+use crate::intrinsics::AtomicOrdering as AO;
 use crate::{fmt, intrinsics};
 
 trait Sealed {}
@@ -3811,9 +3812,9 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
     // SAFETY: the caller must uphold the safety contract for `atomic_store`.
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_store_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_store::<T, { AO::Relaxed }>(dst, val),
-            Release => intrinsics::atomic_store_release(dst, val),
+            Release => intrinsics::atomic_store::<T, { AO::Release }>(dst, val),
-            SeqCst => intrinsics::atomic_store_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_store::<T, { AO::SeqCst }>(dst, val),
             Acquire => panic!("there is no such thing as an acquire store"),
             AcqRel => panic!("there is no such thing as an acquire-release store"),
         }
@@ -3823,13 +3824,12 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
 #[inline]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
 unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
-    use intrinsics::AtomicOrdering;
     // SAFETY: the caller must uphold the safety contract for `atomic_load`.
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_load::<T, { AtomicOrdering::Relaxed }>(dst),
+            Relaxed => intrinsics::atomic_load::<T, { AO::Relaxed }>(dst),
-            Acquire => intrinsics::atomic_load::<T, { AtomicOrdering::Acquire }>(dst),
+            Acquire => intrinsics::atomic_load::<T, { AO::Acquire }>(dst),
-            SeqCst => intrinsics::atomic_load::<T, { AtomicOrdering::SeqCst }>(dst),
+            SeqCst => intrinsics::atomic_load::<T, { AO::SeqCst }>(dst),
             Release => panic!("there is no such thing as a release load"),
             AcqRel => panic!("there is no such thing as an acquire-release load"),
         }
@@ -3843,11 +3843,11 @@ unsafe fn atomic_swap<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_swap`.
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_xchg_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_xchg::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_xchg_acquire(dst, val),
+            Acquire => intrinsics::atomic_xchg::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_xchg_release(dst, val),
+            Release => intrinsics::atomic_xchg::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_xchg_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_xchg::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_xchg_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_xchg::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
@@ -3860,11 +3860,11 @@ unsafe fn atomic_add<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_add`.
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_xadd_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_xadd::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_xadd_acquire(dst, val),
+            Acquire => intrinsics::atomic_xadd::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_xadd_release(dst, val),
+            Release => intrinsics::atomic_xadd::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_xadd_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_xadd::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_xadd_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_xadd::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
@@ -3877,11 +3877,11 @@ unsafe fn atomic_sub<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_sub`.
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_xsub_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_xsub::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_xsub_acquire(dst, val),
+            Acquire => intrinsics::atomic_xsub::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_xsub_release(dst, val),
+            Release => intrinsics::atomic_xsub::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_xsub_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_xsub::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_xsub_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_xsub::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
@@ -3902,21 +3902,51 @@ pub unsafe fn atomic_compare_exchange<T: Copy>(
     // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange`.
     let (val, ok) = unsafe {
         match (success, failure) {
-            (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed_relaxed(dst, old, new),
+            (Relaxed, Relaxed) => {
-            (Relaxed, Acquire) => intrinsics::atomic_cxchg_relaxed_acquire(dst, old, new),
+                intrinsics::atomic_cxchg::<T, { AO::Relaxed }, { AO::Relaxed }>(dst, old, new)
-            (Relaxed, SeqCst) => intrinsics::atomic_cxchg_relaxed_seqcst(dst, old, new),
+            }
-            (Acquire, Relaxed) => intrinsics::atomic_cxchg_acquire_relaxed(dst, old, new),
+            (Relaxed, Acquire) => {
-            (Acquire, Acquire) => intrinsics::atomic_cxchg_acquire_acquire(dst, old, new),
+                intrinsics::atomic_cxchg::<T, { AO::Relaxed }, { AO::Acquire }>(dst, old, new)
-            (Acquire, SeqCst) => intrinsics::atomic_cxchg_acquire_seqcst(dst, old, new),
+            }
-            (Release, Relaxed) => intrinsics::atomic_cxchg_release_relaxed(dst, old, new),
+            (Relaxed, SeqCst) => {
-            (Release, Acquire) => intrinsics::atomic_cxchg_release_acquire(dst, old, new),
+                intrinsics::atomic_cxchg::<T, { AO::Relaxed }, { AO::SeqCst }>(dst, old, new)
-            (Release, SeqCst) => intrinsics::atomic_cxchg_release_seqcst(dst, old, new),
+            }
-            (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_relaxed(dst, old, new),
+            (Acquire, Relaxed) => {
-            (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel_acquire(dst, old, new),
+                intrinsics::atomic_cxchg::<T, { AO::Acquire }, { AO::Relaxed }>(dst, old, new)
-            (AcqRel, SeqCst) => intrinsics::atomic_cxchg_acqrel_seqcst(dst, old, new),
+            }
-            (SeqCst, Relaxed) => intrinsics::atomic_cxchg_seqcst_relaxed(dst, old, new),
+            (Acquire, Acquire) => {
-            (SeqCst, Acquire) => intrinsics::atomic_cxchg_seqcst_acquire(dst, old, new),
+                intrinsics::atomic_cxchg::<T, { AO::Acquire }, { AO::Acquire }>(dst, old, new)
-            (SeqCst, SeqCst) => intrinsics::atomic_cxchg_seqcst_seqcst(dst, old, new),
+            }
+            (Acquire, SeqCst) => {
+                intrinsics::atomic_cxchg::<T, { AO::Acquire }, { AO::SeqCst }>(dst, old, new)
+            }
+            (Release, Relaxed) => {
+                intrinsics::atomic_cxchg::<T, { AO::Release }, { AO::Relaxed }>(dst, old, new)
+            }
+            (Release, Acquire) => {
+                intrinsics::atomic_cxchg::<T, { AO::Release }, { AO::Acquire }>(dst, old, new)
+            }
+            (Release, SeqCst) => {
+                intrinsics::atomic_cxchg::<T, { AO::Release }, { AO::SeqCst }>(dst, old, new)
+            }
+            (AcqRel, Relaxed) => {
+                intrinsics::atomic_cxchg::<T, { AO::AcqRel }, { AO::Relaxed }>(dst, old, new)
+            }
+            (AcqRel, Acquire) => {
+                intrinsics::atomic_cxchg::<T, { AO::AcqRel }, { AO::Acquire }>(dst, old, new)
+            }
+            (AcqRel, SeqCst) => {
+                intrinsics::atomic_cxchg::<T, { AO::AcqRel }, { AO::SeqCst }>(dst, old, new)
+            }
+            (SeqCst, Relaxed) => {
+                intrinsics::atomic_cxchg::<T, { AO::SeqCst }, { AO::Relaxed }>(dst, old, new)
+            }
+            (SeqCst, Acquire) => {
+                intrinsics::atomic_cxchg::<T, { AO::SeqCst }, { AO::Acquire }>(dst, old, new)
+            }
+            (SeqCst, SeqCst) => {
+                intrinsics::atomic_cxchg::<T, { AO::SeqCst }, { AO::SeqCst }>(dst, old, new)
+            }
             (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
             (_, Release) => panic!("there is no such thing as a release failure ordering"),
         }
@@ -3937,21 +3967,51 @@ unsafe fn atomic_compare_exchange_weak<T: Copy>(
     // SAFETY: the caller must uphold the safety contract for `atomic_compare_exchange_weak`.
     let (val, ok) = unsafe {
         match (success, failure) {
-            (Relaxed, Relaxed) => intrinsics::atomic_cxchgweak_relaxed_relaxed(dst, old, new),
+            (Relaxed, Relaxed) => {
-            (Relaxed, Acquire) => intrinsics::atomic_cxchgweak_relaxed_acquire(dst, old, new),
+                intrinsics::atomic_cxchgweak::<T, { AO::Relaxed }, { AO::Relaxed }>(dst, old, new)
-            (Relaxed, SeqCst) => intrinsics::atomic_cxchgweak_relaxed_seqcst(dst, old, new),
+            }
-            (Acquire, Relaxed) => intrinsics::atomic_cxchgweak_acquire_relaxed(dst, old, new),
+            (Relaxed, Acquire) => {
-            (Acquire, Acquire) => intrinsics::atomic_cxchgweak_acquire_acquire(dst, old, new),
+                intrinsics::atomic_cxchgweak::<T, { AO::Relaxed }, { AO::Acquire }>(dst, old, new)
-            (Acquire, SeqCst) => intrinsics::atomic_cxchgweak_acquire_seqcst(dst, old, new),
+            }
-            (Release, Relaxed) => intrinsics::atomic_cxchgweak_release_relaxed(dst, old, new),
+            (Relaxed, SeqCst) => {
-            (Release, Acquire) => intrinsics::atomic_cxchgweak_release_acquire(dst, old, new),
+                intrinsics::atomic_cxchgweak::<T, { AO::Relaxed }, { AO::SeqCst }>(dst, old, new)
-            (Release, SeqCst) => intrinsics::atomic_cxchgweak_release_seqcst(dst, old, new),
+            }
-            (AcqRel, Relaxed) => intrinsics::atomic_cxchgweak_acqrel_relaxed(dst, old, new),
+            (Acquire, Relaxed) => {
-            (AcqRel, Acquire) => intrinsics::atomic_cxchgweak_acqrel_acquire(dst, old, new),
+                intrinsics::atomic_cxchgweak::<T, { AO::Acquire }, { AO::Relaxed }>(dst, old, new)
-            (AcqRel, SeqCst) => intrinsics::atomic_cxchgweak_acqrel_seqcst(dst, old, new),
+            }
-            (SeqCst, Relaxed) => intrinsics::atomic_cxchgweak_seqcst_relaxed(dst, old, new),
+            (Acquire, Acquire) => {
-            (SeqCst, Acquire) => intrinsics::atomic_cxchgweak_seqcst_acquire(dst, old, new),
+                intrinsics::atomic_cxchgweak::<T, { AO::Acquire }, { AO::Acquire }>(dst, old, new)
-            (SeqCst, SeqCst) => intrinsics::atomic_cxchgweak_seqcst_seqcst(dst, old, new),
+            }
+            (Acquire, SeqCst) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::Acquire }, { AO::SeqCst }>(dst, old, new)
+            }
+            (Release, Relaxed) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::Release }, { AO::Relaxed }>(dst, old, new)
+            }
+            (Release, Acquire) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::Release }, { AO::Acquire }>(dst, old, new)
+            }
+            (Release, SeqCst) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::Release }, { AO::SeqCst }>(dst, old, new)
+            }
+            (AcqRel, Relaxed) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::AcqRel }, { AO::Relaxed }>(dst, old, new)
+            }
+            (AcqRel, Acquire) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::AcqRel }, { AO::Acquire }>(dst, old, new)
+            }
+            (AcqRel, SeqCst) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::AcqRel }, { AO::SeqCst }>(dst, old, new)
+            }
+            (SeqCst, Relaxed) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::SeqCst }, { AO::Relaxed }>(dst, old, new)
+            }
+            (SeqCst, Acquire) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::SeqCst }, { AO::Acquire }>(dst, old, new)
+            }
+            (SeqCst, SeqCst) => {
+                intrinsics::atomic_cxchgweak::<T, { AO::SeqCst }, { AO::SeqCst }>(dst, old, new)
+            }
             (_, AcqRel) => panic!("there is no such thing as an acquire-release failure ordering"),
             (_, Release) => panic!("there is no such thing as a release failure ordering"),
         }
@@ -3966,11 +4026,11 @@ unsafe fn atomic_and<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_and`
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_and_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_and::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_and_acquire(dst, val),
+            Acquire => intrinsics::atomic_and::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_and_release(dst, val),
+            Release => intrinsics::atomic_and::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_and_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_and::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_and_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_and::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
@@ -3982,11 +4042,11 @@ unsafe fn atomic_nand<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_nand`
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_nand_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_nand::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_nand_acquire(dst, val),
+            Acquire => intrinsics::atomic_nand::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_nand_release(dst, val),
+            Release => intrinsics::atomic_nand::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_nand_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_nand::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_nand_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_nand::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
@@ -3998,11 +4058,11 @@ unsafe fn atomic_or<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_or`
     unsafe {
         match order {
-            SeqCst => intrinsics::atomic_or_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_or::<T, { AO::SeqCst }>(dst, val),
-            Acquire => intrinsics::atomic_or_acquire(dst, val),
+            Acquire => intrinsics::atomic_or::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_or_release(dst, val),
+            Release => intrinsics::atomic_or::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_or_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_or::<T, { AO::AcqRel }>(dst, val),
-            Relaxed => intrinsics::atomic_or_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_or::<T, { AO::Relaxed }>(dst, val),
         }
     }
 }
@@ -4014,16 +4074,16 @@ unsafe fn atomic_xor<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_xor`
     unsafe {
         match order {
-            SeqCst => intrinsics::atomic_xor_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_xor::<T, { AO::SeqCst }>(dst, val),
-            Acquire => intrinsics::atomic_xor_acquire(dst, val),
+            Acquire => intrinsics::atomic_xor::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_xor_release(dst, val),
+            Release => intrinsics::atomic_xor::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_xor_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_xor::<T, { AO::AcqRel }>(dst, val),
-            Relaxed => intrinsics::atomic_xor_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_xor::<T, { AO::Relaxed }>(dst, val),
         }
     }
 }
 
-/// returns the max value (signed comparison)
+/// Updates `*dst` to the max value of `val` and the old value (signed comparison)
 #[inline]
 #[cfg(target_has_atomic)]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@@ -4031,16 +4091,16 @@ unsafe fn atomic_max<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_max`
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_max_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_max::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_max_acquire(dst, val),
+            Acquire => intrinsics::atomic_max::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_max_release(dst, val),
+            Release => intrinsics::atomic_max::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_max_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_max::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_max_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_max::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
 
-/// returns the min value (signed comparison)
+/// Updates `*dst` to the min value of `val` and the old value (signed comparison)
 #[inline]
 #[cfg(target_has_atomic)]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@@ -4048,16 +4108,16 @@ unsafe fn atomic_min<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_min`
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_min_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_min::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_min_acquire(dst, val),
+            Acquire => intrinsics::atomic_min::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_min_release(dst, val),
+            Release => intrinsics::atomic_min::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_min_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_min::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_min_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_min::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
 
-/// returns the max value (unsigned comparison)
+/// Updates `*dst` to the max value of `val` and the old value (unsigned comparison)
 #[inline]
 #[cfg(target_has_atomic)]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@@ -4065,16 +4125,16 @@ unsafe fn atomic_umax<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_umax`
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_umax_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_umax::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_umax_acquire(dst, val),
+            Acquire => intrinsics::atomic_umax::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_umax_release(dst, val),
+            Release => intrinsics::atomic_umax::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_umax_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_umax::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_umax_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_umax::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
 
-/// returns the min value (unsigned comparison)
+/// Updates `*dst` to the min value of `val` and the old value (unsigned comparison)
 #[inline]
 #[cfg(target_has_atomic)]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
@@ -4082,11 +4142,11 @@ unsafe fn atomic_umin<T: Copy>(dst: *mut T, val: T, order: Ordering) -> T {
     // SAFETY: the caller must uphold the safety contract for `atomic_umin`
     unsafe {
         match order {
-            Relaxed => intrinsics::atomic_umin_relaxed(dst, val),
+            Relaxed => intrinsics::atomic_umin::<T, { AO::Relaxed }>(dst, val),
-            Acquire => intrinsics::atomic_umin_acquire(dst, val),
+            Acquire => intrinsics::atomic_umin::<T, { AO::Acquire }>(dst, val),
-            Release => intrinsics::atomic_umin_release(dst, val),
+            Release => intrinsics::atomic_umin::<T, { AO::Release }>(dst, val),
-            AcqRel => intrinsics::atomic_umin_acqrel(dst, val),
+            AcqRel => intrinsics::atomic_umin::<T, { AO::AcqRel }>(dst, val),
-            SeqCst => intrinsics::atomic_umin_seqcst(dst, val),
+            SeqCst => intrinsics::atomic_umin::<T, { AO::SeqCst }>(dst, val),
         }
     }
 }
@@ -4178,10 +4238,10 @@ pub fn fence(order: Ordering) {
     // SAFETY: using an atomic fence is safe.
     unsafe {
         match order {
-            Acquire => intrinsics::atomic_fence_acquire(),
+            Acquire => intrinsics::atomic_fence::<{ AO::Acquire }>(),
-            Release => intrinsics::atomic_fence_release(),
+            Release => intrinsics::atomic_fence::<{ AO::Release }>(),
-            AcqRel => intrinsics::atomic_fence_acqrel(),
+            AcqRel => intrinsics::atomic_fence::<{ AO::AcqRel }>(),
-            SeqCst => intrinsics::atomic_fence_seqcst(),
+            SeqCst => intrinsics::atomic_fence::<{ AO::SeqCst }>(),
             Relaxed => panic!("there is no such thing as a relaxed fence"),
         }
     }
@@ -4256,11 +4316,11 @@ pub fn compiler_fence(order: Ordering) {
     // SAFETY: using an atomic fence is safe.
     unsafe {
         match order {
-            Acquire => intrinsics::atomic_singlethreadfence_acquire(),
+            Acquire => intrinsics::atomic_singlethreadfence::<{ AO::Acquire }>(),
-            Release => intrinsics::atomic_singlethreadfence_release(),
+            Release => intrinsics::atomic_singlethreadfence::<{ AO::Release }>(),
-            AcqRel => intrinsics::atomic_singlethreadfence_acqrel(),
+            AcqRel => intrinsics::atomic_singlethreadfence::<{ AO::AcqRel }>(),
-            SeqCst => intrinsics::atomic_singlethreadfence_seqcst(),
+            SeqCst => intrinsics::atomic_singlethreadfence::<{ AO::SeqCst }>(),
-            Relaxed => panic!("there is no such thing as a relaxed compiler fence"),
+            Relaxed => panic!("there is no such thing as a relaxed fence"),
         }
     }
 }

library/panic_unwind/src/seh.rs

@@ -291,7 +291,7 @@ cfg_if::cfg_if! {
 }
 
 pub(crate) unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
-    use core::intrinsics::atomic_store_seqcst;
+    use core::intrinsics::{AtomicOrdering, atomic_store};
 
     // _CxxThrowException executes entirely on this stack frame, so there's no
     // need to otherwise transfer `data` to the heap. We just pass a stack
@@ -325,23 +325,23 @@ pub(crate) unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
     // In any case, we basically need to do something like this until we can
     // express more operations in statics (and we may never be able to).
     unsafe {
-        atomic_store_seqcst(
+        atomic_store::<_, { AtomicOrdering::SeqCst }>(
             (&raw mut THROW_INFO.pmfnUnwind).cast(),
             ptr_t::new(exception_cleanup as *mut u8).raw(),
         );
-        atomic_store_seqcst(
+        atomic_store::<_, { AtomicOrdering::SeqCst }>(
             (&raw mut THROW_INFO.pCatchableTypeArray).cast(),
             ptr_t::new((&raw mut CATCHABLE_TYPE_ARRAY).cast()).raw(),
         );
-        atomic_store_seqcst(
+        atomic_store::<_, { AtomicOrdering::SeqCst }>(
             (&raw mut CATCHABLE_TYPE_ARRAY.arrayOfCatchableTypes[0]).cast(),
             ptr_t::new((&raw mut CATCHABLE_TYPE).cast()).raw(),
         );
-        atomic_store_seqcst(
+        atomic_store::<_, { AtomicOrdering::SeqCst }>(
             (&raw mut CATCHABLE_TYPE.pType).cast(),
             ptr_t::new((&raw mut TYPE_DESCRIPTOR).cast()).raw(),
         );
-        atomic_store_seqcst(
+        atomic_store::<_, { AtomicOrdering::SeqCst }>(
             (&raw mut CATCHABLE_TYPE.copyFunction).cast(),
             ptr_t::new(exception_copy as *mut u8).raw(),
         );

src/tools/miri/src/intrinsics/atomic.rs

@@ -26,108 +26,131 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
     ) -> InterpResult<'tcx, EmulateItemResult> {
         let this = self.eval_context_mut();
 
-        let intrinsic_structure: Vec<_> = intrinsic_name.split('_').collect();
+        let get_ord_at = |i: usize| {
+            let ordering = generic_args.const_at(i).to_value();
+            ordering.valtree.unwrap_branch()[0].unwrap_leaf().to_atomic_ordering()
+        };
 
-        fn read_ord(ord: &str) -> AtomicReadOrd {
+        fn read_ord(ord: AtomicOrdering) -> AtomicReadOrd {
             match ord {
-                "seqcst" => AtomicReadOrd::SeqCst,
-                "acquire" => AtomicReadOrd::Acquire,
-                "relaxed" => AtomicReadOrd::Relaxed,
-                _ => panic!("invalid read ordering `{ord}`"),
-            }
-        }
-
-        fn read_ord_const_generic(o: AtomicOrdering) -> AtomicReadOrd {
-            match o {
                 AtomicOrdering::SeqCst => AtomicReadOrd::SeqCst,
                 AtomicOrdering::Acquire => AtomicReadOrd::Acquire,
                 AtomicOrdering::Relaxed => AtomicReadOrd::Relaxed,
-                _ => panic!("invalid read ordering `{o:?}`"),
+                _ => panic!("invalid read ordering `{ord:?}`"),
             }
         }
 
-        fn write_ord(ord: &str) -> AtomicWriteOrd {
+        fn write_ord(ord: AtomicOrdering) -> AtomicWriteOrd {
             match ord {
-                "seqcst" => AtomicWriteOrd::SeqCst,
+                AtomicOrdering::SeqCst => AtomicWriteOrd::SeqCst,
-                "release" => AtomicWriteOrd::Release,
+                AtomicOrdering::Release => AtomicWriteOrd::Release,
-                "relaxed" => AtomicWriteOrd::Relaxed,
+                AtomicOrdering::Relaxed => AtomicWriteOrd::Relaxed,
-                _ => panic!("invalid write ordering `{ord}`"),
+                _ => panic!("invalid write ordering `{ord:?}`"),
             }
         }
 
-        fn rw_ord(ord: &str) -> AtomicRwOrd {
+        fn rw_ord(ord: AtomicOrdering) -> AtomicRwOrd {
             match ord {
-                "seqcst" => AtomicRwOrd::SeqCst,
+                AtomicOrdering::SeqCst => AtomicRwOrd::SeqCst,
-                "acqrel" => AtomicRwOrd::AcqRel,
+                AtomicOrdering::AcqRel => AtomicRwOrd::AcqRel,
-                "acquire" => AtomicRwOrd::Acquire,
+                AtomicOrdering::Acquire => AtomicRwOrd::Acquire,
-                "release" => AtomicRwOrd::Release,
+                AtomicOrdering::Release => AtomicRwOrd::Release,
-                "relaxed" => AtomicRwOrd::Relaxed,
+                AtomicOrdering::Relaxed => AtomicRwOrd::Relaxed,
-                _ => panic!("invalid read-write ordering `{ord}`"),
             }
         }
 
-        fn fence_ord(ord: &str) -> AtomicFenceOrd {
+        fn fence_ord(ord: AtomicOrdering) -> AtomicFenceOrd {
             match ord {
-                "seqcst" => AtomicFenceOrd::SeqCst,
+                AtomicOrdering::SeqCst => AtomicFenceOrd::SeqCst,
-                "acqrel" => AtomicFenceOrd::AcqRel,
+                AtomicOrdering::AcqRel => AtomicFenceOrd::AcqRel,
-                "acquire" => AtomicFenceOrd::Acquire,
+                AtomicOrdering::Acquire => AtomicFenceOrd::Acquire,
-                "release" => AtomicFenceOrd::Release,
+                AtomicOrdering::Release => AtomicFenceOrd::Release,
-                _ => panic!("invalid fence ordering `{ord}`"),
+                _ => panic!("invalid fence ordering `{ord:?}`"),
             }
         }
 
-        match &*intrinsic_structure {
+        match intrinsic_name {
-            // New-style intrinsics that use const generics
+            "load" => {
-            ["load"] => {
+                let ord = get_ord_at(1);
-                let ordering = generic_args.const_at(1).to_value();
+                this.atomic_load(args, dest, read_ord(ord))?;
-                let ordering =
-                    ordering.valtree.unwrap_branch()[0].unwrap_leaf().to_atomic_ordering();
-                this.atomic_load(args, dest, read_ord_const_generic(ordering))?;
             }
 
-            // Old-style intrinsics that have the ordering in the intrinsic name
+            "store" => {
-            ["store", ord] => this.atomic_store(args, write_ord(ord))?,
+                let ord = get_ord_at(1);
+                this.atomic_store(args, write_ord(ord))?
+            }
 
-            ["fence", ord] => this.atomic_fence_intrinsic(args, fence_ord(ord))?,
+            "fence" => {
-            ["singlethreadfence", ord] => this.compiler_fence_intrinsic(args, fence_ord(ord))?,
+                let ord = get_ord_at(0);
+                this.atomic_fence_intrinsic(args, fence_ord(ord))?
+            }
+            "singlethreadfence" => {
+                let ord = get_ord_at(0);
+                this.compiler_fence_intrinsic(args, fence_ord(ord))?;
+            }
 
-            ["xchg", ord] => this.atomic_exchange(args, dest, rw_ord(ord))?,
+            "xchg" => {
-            ["cxchg", ord1, ord2] =>
+                let ord = get_ord_at(1);
-                this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?,
+                this.atomic_exchange(args, dest, rw_ord(ord))?;
-            ["cxchgweak", ord1, ord2] =>
+            }
-                this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?,
+            "cxchg" => {
+                let ord1 = get_ord_at(1);
+                let ord2 = get_ord_at(2);
+                this.atomic_compare_exchange(args, dest, rw_ord(ord1), read_ord(ord2))?;
+            }
+            "cxchgweak" => {
+                let ord1 = get_ord_at(1);
+                let ord2 = get_ord_at(2);
+                this.atomic_compare_exchange_weak(args, dest, rw_ord(ord1), read_ord(ord2))?;
+            }
 
-            ["or", ord] =>
+            "or" => {
-                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?,
+                let ord = get_ord_at(1);
-            ["xor", ord] =>
+                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitOr, false), rw_ord(ord))?;
-                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?,
+            }
-            ["and", ord] =>
+            "xor" => {
-                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?,
+                let ord = get_ord_at(1);
-            ["nand", ord] =>
+                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitXor, false), rw_ord(ord))?;
-                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?,
+            }
-            ["xadd", ord] =>
+            "and" => {
-                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?,
+                let ord = get_ord_at(1);
-            ["xsub", ord] =>
+                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, false), rw_ord(ord))?;
-                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?,
+            }
-            ["min", ord] => {
+            "nand" => {
+                let ord = get_ord_at(1);
+                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::BitAnd, true), rw_ord(ord))?;
+            }
+            "xadd" => {
+                let ord = get_ord_at(1);
+                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Add, false), rw_ord(ord))?;
+            }
+            "xsub" => {
+                let ord = get_ord_at(1);
+                this.atomic_rmw_op(args, dest, AtomicOp::MirOp(BinOp::Sub, false), rw_ord(ord))?;
+            }
+            "min" => {
+                let ord = get_ord_at(1);
                 // Later we will use the type to indicate signed vs unsigned,
                 // so make sure it matches the intrinsic name.
                 assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
                 this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
             }
-            ["umin", ord] => {
+            "umin" => {
+                let ord = get_ord_at(1);
                 // Later we will use the type to indicate signed vs unsigned,
                 // so make sure it matches the intrinsic name.
                 assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));
                 this.atomic_rmw_op(args, dest, AtomicOp::Min, rw_ord(ord))?;
             }
-            ["max", ord] => {
+            "max" => {
+                let ord = get_ord_at(1);
                 // Later we will use the type to indicate signed vs unsigned,
                 // so make sure it matches the intrinsic name.
                 assert!(matches!(args[1].layout.ty.kind(), ty::Int(_)));
                 this.atomic_rmw_op(args, dest, AtomicOp::Max, rw_ord(ord))?;
             }
-            ["umax", ord] => {
+            "umax" => {
+                let ord = get_ord_at(1);
                 // Later we will use the type to indicate signed vs unsigned,
                 // so make sure it matches the intrinsic name.
                 assert!(matches!(args[1].layout.ty.kind(), ty::Uint(_)));

tests/run-make/atomic-lock-free/atomic_lock_free.rs

@@ -1,9 +1,20 @@
 #![feature(no_core, intrinsics, lang_items)]
+#![feature(adt_const_params)]
 #![crate_type = "rlib"]
 #![no_core]
 
+pub enum AtomicOrdering {
+    // These values must match the compiler's `AtomicOrdering` defined in
+    // `rustc_middle/src/ty/consts/int.rs`!
+    Relaxed = 0,
+    Release = 1,
+    Acquire = 2,
+    AcqRel = 3,
+    SeqCst = 4,
+}
+
 #[rustc_intrinsic]
-unsafe fn atomic_xadd_seqcst<T>(dst: *mut T, src: T) -> T;
+unsafe fn atomic_xadd<T, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
 
 #[lang = "sized"]
 trait Sized {}
@@ -11,55 +22,58 @@ trait Sized {}
 trait Copy {}
 #[lang = "freeze"]
 trait Freeze {}
+#[lang = "const_param_ty"]
+pub trait ConstParamTy {}
 
 impl<T: ?Sized> Copy for *mut T {}
+impl ConstParamTy for AtomicOrdering {}
 
 #[cfg(target_has_atomic = "8")]
 pub unsafe fn atomic_u8(x: *mut u8) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "8")]
 pub unsafe fn atomic_i8(x: *mut i8) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "16")]
 pub unsafe fn atomic_u16(x: *mut u16) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "16")]
 pub unsafe fn atomic_i16(x: *mut i16) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "32")]
 pub unsafe fn atomic_u32(x: *mut u32) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "32")]
 pub unsafe fn atomic_i32(x: *mut i32) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "64")]
 pub unsafe fn atomic_u64(x: *mut u64) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "64")]
 pub unsafe fn atomic_i64(x: *mut i64) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "128")]
 pub unsafe fn atomic_u128(x: *mut u128) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "128")]
 pub unsafe fn atomic_i128(x: *mut i128) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "ptr")]
 pub unsafe fn atomic_usize(x: *mut usize) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }
 #[cfg(target_has_atomic = "ptr")]
 pub unsafe fn atomic_isize(x: *mut isize) {
-    atomic_xadd_seqcst(x, 1);
+    atomic_xadd::<_, { AtomicOrdering::SeqCst }>(x, 1);
 }

tests/ui/error-codes/E0092.rs

@@ -1,6 +0,0 @@
-#![feature(intrinsics)]
-
-#[rustc_intrinsic]
-unsafe fn atomic_foo(); //~ ERROR E0092
-
-fn main() {}

tests/ui/error-codes/E0092.stderr

@@ -1,9 +0,0 @@
-error[E0092]: unrecognized atomic operation function: `foo`
-  --> $DIR/E0092.rs:4:11
-   |
-LL | unsafe fn atomic_foo();
-   |           ^^^^^^^^^^ unrecognized atomic operation
-
-error: aborting due to 1 previous error
-
-For more information about this error, try `rustc --explain E0092`.

tests/ui/feature-gates/feature-gated-feature-in-macro-arg.stderr

@@ -7,7 +7,7 @@ LL |         #[rustc_intrinsic]
    = help: add `#![feature(intrinsics)]` to the crate attributes to enable
    = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
 
-error[E0133]: call to unsafe function `atomic_fence` is unsafe and requires unsafe function or block
+error[E0133]: call to unsafe function `main::atomic_fence` is unsafe and requires unsafe function or block
   --> $DIR/feature-gated-feature-in-macro-arg.rs:11:9
    |
 LL |         atomic_fence();

tests/ui/intrinsics/auxiliary/cci_intrinsic.rs

@@ -1,11 +1,24 @@
-#![feature(intrinsics)]
+#![feature(intrinsics, adt_const_params)]
 
 pub mod rusti {
+    use std::marker::ConstParamTy;
+
+    #[derive(Debug, ConstParamTy, PartialEq, Eq)]
+    pub enum AtomicOrdering {
+        // These values must match the compiler's `AtomicOrdering` defined in
+        // `rustc_middle/src/ty/consts/int.rs`!
+        Relaxed = 0,
+        Release = 1,
+        Acquire = 2,
+        AcqRel = 3,
+        SeqCst = 4,
+    }
+
     #[rustc_intrinsic]
-    pub unsafe fn atomic_xchg_seqcst<T>(dst: *mut T, src: T) -> T;
+    pub unsafe fn atomic_xchg<T, const ORD: AtomicOrdering>(dst: *mut T, src: T) -> T;
 }
 
 #[inline(always)]
 pub fn atomic_xchg_seqcst(dst: *mut isize, src: isize) -> isize {
-    unsafe { rusti::atomic_xchg_seqcst(dst, src) }
+    unsafe { rusti::atomic_xchg::<_, { rusti::AtomicOrdering::SeqCst }>(dst, src) }
 }

tests/ui/intrinsics/intrinsic-atomics.rs

@@ -1,50 +1,50 @@
 //@ run-pass
 #![feature(core_intrinsics)]
-use std::intrinsics::{self as rusti, AtomicOrdering};
+use std::intrinsics::{self as rusti, AtomicOrdering::*};
 
 pub fn main() {
     unsafe {
         let mut x: Box<_> = Box::new(1);
 
-        assert_eq!(rusti::atomic_load::<_, { AtomicOrdering::SeqCst }>(&*x), 1);
+        assert_eq!(rusti::atomic_load::<_, { SeqCst }>(&*x), 1);
         *x = 5;
-        assert_eq!(rusti::atomic_load::<_, { AtomicOrdering::Acquire }>(&*x), 5);
+        assert_eq!(rusti::atomic_load::<_, { Acquire }>(&*x), 5);
 
-        rusti::atomic_store_seqcst(&mut *x, 3);
+        rusti::atomic_store::<_, { SeqCst }>(&mut *x, 3);
         assert_eq!(*x, 3);
-        rusti::atomic_store_release(&mut *x, 1);
+        rusti::atomic_store::<_, { Release }>(&mut *x, 1);
         assert_eq!(*x, 1);
 
-        assert_eq!(rusti::atomic_cxchg_seqcst_seqcst(&mut *x, 1, 2), (1, true));
+        assert_eq!(rusti::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(&mut *x, 1, 2), (1, true));
         assert_eq!(*x, 2);
 
-        assert_eq!(rusti::atomic_cxchg_acquire_acquire(&mut *x, 1, 3), (2, false));
+        assert_eq!(rusti::atomic_cxchg::<_, { Acquire }, { Acquire }>(&mut *x, 1, 3), (2, false));
         assert_eq!(*x, 2);
 
-        assert_eq!(rusti::atomic_cxchg_release_relaxed(&mut *x, 2, 1), (2, true));
+        assert_eq!(rusti::atomic_cxchg::<_, { Release }, { Relaxed }>(&mut *x, 2, 1), (2, true));
         assert_eq!(*x, 1);
 
-        assert_eq!(rusti::atomic_xchg_seqcst(&mut *x, 0), 1);
+        assert_eq!(rusti::atomic_xchg::<_, { SeqCst }>(&mut *x, 0), 1);
         assert_eq!(*x, 0);
 
-        assert_eq!(rusti::atomic_xchg_acquire(&mut *x, 1), 0);
+        assert_eq!(rusti::atomic_xchg::<_, { Acquire }>(&mut *x, 1), 0);
         assert_eq!(*x, 1);
 
-        assert_eq!(rusti::atomic_xchg_release(&mut *x, 0), 1);
+        assert_eq!(rusti::atomic_xchg::<_, { Release }>(&mut *x, 0), 1);
         assert_eq!(*x, 0);
 
-        assert_eq!(rusti::atomic_xadd_seqcst(&mut *x, 1), 0);
+        assert_eq!(rusti::atomic_xadd::<_, { SeqCst }>(&mut *x, 1), 0);
-        assert_eq!(rusti::atomic_xadd_acquire(&mut *x, 1), 1);
+        assert_eq!(rusti::atomic_xadd::<_, { Acquire }>(&mut *x, 1), 1);
-        assert_eq!(rusti::atomic_xadd_release(&mut *x, 1), 2);
+        assert_eq!(rusti::atomic_xadd::<_, { Release }>(&mut *x, 1), 2);
         assert_eq!(*x, 3);
 
-        assert_eq!(rusti::atomic_xsub_seqcst(&mut *x, 1), 3);
+        assert_eq!(rusti::atomic_xsub::<_, { SeqCst }>(&mut *x, 1), 3);
-        assert_eq!(rusti::atomic_xsub_acquire(&mut *x, 1), 2);
+        assert_eq!(rusti::atomic_xsub::<_, { Acquire }>(&mut *x, 1), 2);
-        assert_eq!(rusti::atomic_xsub_release(&mut *x, 1), 1);
+        assert_eq!(rusti::atomic_xsub::<_, { Release }>(&mut *x, 1), 1);
         assert_eq!(*x, 0);
 
         loop {
-            let res = rusti::atomic_cxchgweak_seqcst_seqcst(&mut *x, 0, 1);
+            let res = rusti::atomic_cxchgweak::<_, { SeqCst }, { SeqCst }>(&mut *x, 0, 1);
             assert_eq!(res.0, 0);
             if res.1 {
                 break;
@@ -53,7 +53,7 @@ pub fn main() {
         assert_eq!(*x, 1);
 
         loop {
-            let res = rusti::atomic_cxchgweak_acquire_acquire(&mut *x, 1, 2);
+            let res = rusti::atomic_cxchgweak::<_, { Acquire }, { Acquire }>(&mut *x, 1, 2);
             assert_eq!(res.0, 1);
             if res.1 {
                 break;
@@ -62,7 +62,7 @@ pub fn main() {
         assert_eq!(*x, 2);
 
         loop {
-            let res = rusti::atomic_cxchgweak_release_relaxed(&mut *x, 2, 3);
+            let res = rusti::atomic_cxchgweak::<_, { Release }, { Relaxed }>(&mut *x, 2, 3);
             assert_eq!(res.0, 2);
             if res.1 {
                 break;

tests/ui/intrinsics/non-integer-atomic.rs

@@ -4,7 +4,7 @@
 #![allow(warnings)]
 #![crate_type = "rlib"]
 
-use std::intrinsics::{self, AtomicOrdering};
+use std::intrinsics::{self, AtomicOrdering::*};
 
 #[derive(Copy, Clone)]
 pub struct Foo(i64);
@@ -12,81 +12,81 @@ pub type Bar = &'static dyn Fn();
 pub type Quux = [u8; 100];
 
 pub unsafe fn test_bool_load(p: &mut bool, v: bool) {
-    intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+    intrinsics::atomic_load::<_, { SeqCst }>(p);
     //~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `bool`
 }
 
 pub unsafe fn test_bool_store(p: &mut bool, v: bool) {
-    intrinsics::atomic_store_seqcst(p, v);
+    intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-    //~^ ERROR `atomic_store_seqcst` intrinsic: expected basic integer type, found `bool`
+    //~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `bool`
 }
 
 pub unsafe fn test_bool_xchg(p: &mut bool, v: bool) {
-    intrinsics::atomic_xchg_seqcst(p, v);
+    intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-    //~^ ERROR `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `bool`
+    //~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `bool`
 }
 
 pub unsafe fn test_bool_cxchg(p: &mut bool, v: bool) {
-    intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+    intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-    //~^ ERROR `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `bool`
+    //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `bool`
 }
 
 pub unsafe fn test_Foo_load(p: &mut Foo, v: Foo) {
-    intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+    intrinsics::atomic_load::<_, { SeqCst }>(p);
     //~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `Foo`
 }
 
 pub unsafe fn test_Foo_store(p: &mut Foo, v: Foo) {
-    intrinsics::atomic_store_seqcst(p, v);
+    intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-    //~^ ERROR `atomic_store_seqcst` intrinsic: expected basic integer type, found `Foo`
+    //~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `Foo`
 }
 
 pub unsafe fn test_Foo_xchg(p: &mut Foo, v: Foo) {
-    intrinsics::atomic_xchg_seqcst(p, v);
+    intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-    //~^ ERROR `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `Foo`
+    //~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `Foo`
 }
 
 pub unsafe fn test_Foo_cxchg(p: &mut Foo, v: Foo) {
-    intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+    intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-    //~^ ERROR `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `Foo`
+    //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `Foo`
 }
 
 pub unsafe fn test_Bar_load(p: &mut Bar, v: Bar) {
-    intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+    intrinsics::atomic_load::<_, { SeqCst }>(p);
     //~^ ERROR expected basic integer type, found `&dyn Fn()`
 }
 
 pub unsafe fn test_Bar_store(p: &mut Bar, v: Bar) {
-    intrinsics::atomic_store_seqcst(p, v);
+    intrinsics::atomic_store::<_, { SeqCst }>(p, v);
     //~^ ERROR expected basic integer type, found `&dyn Fn()`
 }
 
 pub unsafe fn test_Bar_xchg(p: &mut Bar, v: Bar) {
-    intrinsics::atomic_xchg_seqcst(p, v);
+    intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
     //~^ ERROR expected basic integer type, found `&dyn Fn()`
 }
 
 pub unsafe fn test_Bar_cxchg(p: &mut Bar, v: Bar) {
-    intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+    intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
     //~^ ERROR expected basic integer type, found `&dyn Fn()`
 }
 
 pub unsafe fn test_Quux_load(p: &mut Quux, v: Quux) {
-    intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+    intrinsics::atomic_load::<_, { SeqCst }>(p);
     //~^ ERROR `atomic_load` intrinsic: expected basic integer type, found `[u8; 100]`
 }
 
 pub unsafe fn test_Quux_store(p: &mut Quux, v: Quux) {
-    intrinsics::atomic_store_seqcst(p, v);
+    intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-    //~^ ERROR `atomic_store_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
+    //~^ ERROR `atomic_store` intrinsic: expected basic integer type, found `[u8; 100]`
 }
 
 pub unsafe fn test_Quux_xchg(p: &mut Quux, v: Quux) {
-    intrinsics::atomic_xchg_seqcst(p, v);
+    intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-    //~^ ERROR `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
+    //~^ ERROR `atomic_xchg` intrinsic: expected basic integer type, found `[u8; 100]`
 }
 
 pub unsafe fn test_Quux_cxchg(p: &mut Quux, v: Quux) {
-    intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+    intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-    //~^ ERROR `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
+    //~^ ERROR `atomic_cxchg` intrinsic: expected basic integer type, found `[u8; 100]`
 }

tests/ui/intrinsics/non-integer-atomic.stderr

@@ -1,98 +1,98 @@
 error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `bool`
   --> $DIR/non-integer-atomic.rs:15:5
    |
-LL |     intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+LL |     intrinsics::atomic_load::<_, { SeqCst }>(p);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `bool`
+error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `bool`
   --> $DIR/non-integer-atomic.rs:20:5
    |
-LL |     intrinsics::atomic_store_seqcst(p, v);
+LL |     intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `bool`
+error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `bool`
   --> $DIR/non-integer-atomic.rs:25:5
    |
-LL |     intrinsics::atomic_xchg_seqcst(p, v);
+LL |     intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `bool`
+error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `bool`
   --> $DIR/non-integer-atomic.rs:30:5
    |
-LL |     intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+LL |     intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `Foo`
   --> $DIR/non-integer-atomic.rs:35:5
    |
-LL |     intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+LL |     intrinsics::atomic_load::<_, { SeqCst }>(p);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `Foo`
+error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `Foo`
   --> $DIR/non-integer-atomic.rs:40:5
    |
-LL |     intrinsics::atomic_store_seqcst(p, v);
+LL |     intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `Foo`
+error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `Foo`
   --> $DIR/non-integer-atomic.rs:45:5
    |
-LL |     intrinsics::atomic_xchg_seqcst(p, v);
+LL |     intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `Foo`
+error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `Foo`
   --> $DIR/non-integer-atomic.rs:50:5
    |
-LL |     intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+LL |     intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `&dyn Fn()`
   --> $DIR/non-integer-atomic.rs:55:5
    |
-LL |     intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+LL |     intrinsics::atomic_load::<_, { SeqCst }>(p);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `&dyn Fn()`
+error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `&dyn Fn()`
   --> $DIR/non-integer-atomic.rs:60:5
    |
-LL |     intrinsics::atomic_store_seqcst(p, v);
+LL |     intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `&dyn Fn()`
+error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `&dyn Fn()`
   --> $DIR/non-integer-atomic.rs:65:5
    |
-LL |     intrinsics::atomic_xchg_seqcst(p, v);
+LL |     intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `&dyn Fn()`
+error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `&dyn Fn()`
   --> $DIR/non-integer-atomic.rs:70:5
    |
-LL |     intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+LL |     intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `atomic_load` intrinsic: expected basic integer type, found `[u8; 100]`
   --> $DIR/non-integer-atomic.rs:75:5
    |
-LL |     intrinsics::atomic_load::<_, { AtomicOrdering::SeqCst }>(p);
+LL |     intrinsics::atomic_load::<_, { SeqCst }>(p);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_store_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
+error[E0511]: invalid monomorphization of `atomic_store` intrinsic: expected basic integer type, found `[u8; 100]`
   --> $DIR/non-integer-atomic.rs:80:5
    |
-LL |     intrinsics::atomic_store_seqcst(p, v);
+LL |     intrinsics::atomic_store::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_xchg_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
+error[E0511]: invalid monomorphization of `atomic_xchg` intrinsic: expected basic integer type, found `[u8; 100]`
   --> $DIR/non-integer-atomic.rs:85:5
    |
-LL |     intrinsics::atomic_xchg_seqcst(p, v);
+LL |     intrinsics::atomic_xchg::<_, { SeqCst }>(p, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error[E0511]: invalid monomorphization of `atomic_cxchg_seqcst_seqcst` intrinsic: expected basic integer type, found `[u8; 100]`
+error[E0511]: invalid monomorphization of `atomic_cxchg` intrinsic: expected basic integer type, found `[u8; 100]`
   --> $DIR/non-integer-atomic.rs:90:5
    |
-LL |     intrinsics::atomic_cxchg_seqcst_seqcst(p, v, v);
+LL |     intrinsics::atomic_cxchg::<_, { SeqCst }, { SeqCst }>(p, v, v);
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: aborting due to 16 previous errors