Add SVE support to stdarch-verify

Co-authored-by: Jamie Cunliffe <Jamie.Cunliffe@arm.com>
Co-authored-by: Jacob Bramley <jacob.bramley@arm.com>
Co-authored-by: Luca Vizzarro <Luca.Vizzarro@arm.com>

library/stdarch/crates/stdarch-verify/src/lib.rs

@@ -45,7 +45,9 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream {
     for &mut (ref mut file, ref path) in &mut files {
         for mut item in file.items.drain(..) {
             match item {
-                syn::Item::Fn(f) => functions.push((f, path)),
+                syn::Item::Fn(f) => {
+                    functions.push((f, path));
+                }
                 syn::Item::Mod(ref mut m) => {
                     if let Some(ref mut m) = m.content {
                         for i in m.1.drain(..) {
@@ -71,12 +73,9 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream {
     assert!(!tests.is_empty());
 
     functions.retain(|(f, _)| {
-        if let syn::Visibility::Public(_) = f.vis {
+        matches!(f.vis, syn::Visibility::Public(_))
-            if f.sig.unsafety.is_some() {
+            // Many SVE intrinsics are safe
-                return true;
+            && (f.sig.unsafety.is_some() || f.sig.ident.to_string().starts_with("sv"))
-            }
-        }
-        false
     });
     assert!(!functions.is_empty());
 
@@ -99,7 +98,7 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream {
             for generic in f.sig.generics.params.iter() {
                 match *generic {
                     syn::GenericParam::Const(ref c) => const_arguments.push(to_type(&c.ty)),
-                    syn::GenericParam::Type(ref _t) => (),
+                    syn::GenericParam::Type(_) => (),
                     _ => panic!("invalid generic argument on {name}"),
                 };
             }
@@ -118,25 +117,31 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream {
             };
 
             let required_const = find_required_const("rustc_args_required_const", &f.attrs);
-            let mut legacy_const_generics =
+            let mut const_generics_indices =
                 find_required_const("rustc_legacy_const_generics", &f.attrs);
-            if !required_const.is_empty() && !legacy_const_generics.is_empty() {
+            if !required_const.is_empty() && !const_generics_indices.is_empty() {
                 panic!(
                     "Can't have both #[rustc_args_required_const] and \
                      #[rustc_legacy_const_generics]"
                 );
             }
 
+            // Newer intrinsics don't have legacy support - assume they belong at the end of the argument list
+            if required_const.is_empty() && const_generics_indices.is_empty() {
+                const_generics_indices =
+                    (arguments.len()..(arguments.len() + const_arguments.len())).collect();
+            }
+
             // The list of required consts, used to verify the arguments, comes from either the
             // `rustc_args_required_const` or the `rustc_legacy_const_generics` attribute.
             let required_const = if required_const.is_empty() {
-                legacy_const_generics.clone()
+                const_generics_indices.clone()
             } else {
                 required_const
             };
 
-            legacy_const_generics.sort();
+            const_generics_indices.sort();
-            for (idx, ty) in legacy_const_generics
+            for (idx, ty) in const_generics_indices
                 .into_iter()
                 .zip(const_arguments.into_iter())
             {
@@ -145,12 +150,12 @@ fn functions(input: TokenStream, dirs: &[&str]) -> TokenStream {
 
             // strip leading underscore from fn name when building a test
             // _mm_foo -> mm_foo such that the test name is test_mm_foo.
-            let test_name_string = format!("{name}");
+            let test_name = name.to_string();
-            let mut test_name_id = test_name_string.as_str();
+            let test_name_id = test_name.trim_start_matches('_');
-            while test_name_id.starts_with('_') {
+            let has_test = tests.contains(&format!("test_{test_name_id}"))
-                test_name_id = &test_name_id[1..];
+                // SVE load/store tests
-            }
+                || tests.iter().any(|t| t.starts_with(&format!("test_{test_name_id}"))
-            let has_test = tests.contains(&format!("test_{test_name_id}"));
+                                        || t.ends_with(&format!("_with_{test_name_id}")));
 
             let doc = find_doc(&f.attrs);
 
@@ -221,8 +226,53 @@ fn to_type(t: &syn::Type) -> proc_macro2::TokenStream {
             "p16" => quote! { &P16 },
             "Ordering" => quote! { &ORDERING },
             "CpuidResult" => quote! { &CPUID },
+            "T" => quote! { &GENERICT },
 
             // arm ...
+            "svbool_t" => quote! { &SVBOOL },
+            "svint8_t" => quote! { &SVI8 },
+            "svint8x2_t" => quote! { &SVI8X2 },
+            "svint8x3_t" => quote! { &SVI8X3 },
+            "svint8x4_t" => quote! { &SVI8X4 },
+            "svint16_t" => quote! { &SVI16 },
+            "svint16x2_t" => quote! { &SVI16X2 },
+            "svint16x3_t" => quote! { &SVI16X3 },
+            "svint16x4_t" => quote! { &SVI16X4 },
+            "svint32_t" => quote! { &SVI32 },
+            "svint32x2_t" => quote! { &SVI32X2 },
+            "svint32x3_t" => quote! { &SVI32X3 },
+            "svint32x4_t" => quote! { &SVI32X4 },
+            "svint64_t" => quote! { &SVI64 },
+            "svint64x2_t" => quote! { &SVI64X2 },
+            "svint64x3_t" => quote! { &SVI64X3 },
+            "svint64x4_t" => quote! { &SVI64X4 },
+            "svuint8_t" => quote! { &SVU8 },
+            "svuint8x2_t" => quote! { &SVU8X2 },
+            "svuint8x3_t" => quote! { &SVU8X3 },
+            "svuint8x4_t" => quote! { &SVU8X4 },
+            "svuint16_t" => quote! { &SVU16 },
+            "svuint16x2_t" => quote! { &SVU16X2 },
+            "svuint16x3_t" => quote! { &SVU16X3 },
+            "svuint16x4_t" => quote! { &SVU16X4 },
+            "svuint32_t" => quote! { &SVU32 },
+            "svuint32x2_t" => quote! { &SVU32X2 },
+            "svuint32x3_t" => quote! { &SVU32X3 },
+            "svuint32x4_t" => quote! { &SVU32X4 },
+            "svuint64_t" => quote! { &SVU64 },
+            "svuint64x2_t" => quote! { &SVU64X2 },
+            "svuint64x3_t" => quote! { &SVU64X3 },
+            "svuint64x4_t" => quote! { &SVU64X4 },
+            "svfloat32_t" => quote! { &SVF32 },
+            "svfloat32x2_t" => quote! { &SVF32X2 },
+            "svfloat32x3_t" => quote! { &SVF32X3 },
+            "svfloat32x4_t" => quote! { &SVF32X4 },
+            "svfloat64_t" => quote! { &SVF64 },
+            "svfloat64x2_t" => quote! { &SVF64X2 },
+            "svfloat64x3_t" => quote! { &SVF64X3 },
+            "svfloat64x4_t" => quote! { &SVF64X4 },
+            "svprfop" => quote! { &SVPRFOP },
+            "svpattern" => quote! { &SVPATTERN },
+
             "int8x4_t" => quote! { &I8X4 },
             "int8x8_t" => quote! { &I8X8 },
             "int8x8x2_t" => quote! { &I8X8X2 },

library/stdarch/crates/stdarch-verify/tests/arm.rs

@@ -27,6 +27,8 @@ static U16: Type = Type::PrimUnsigned(16);
 static U32: Type = Type::PrimUnsigned(32);
 static U64: Type = Type::PrimUnsigned(64);
 static U8: Type = Type::PrimUnsigned(8);
+static BOOL: Type = Type::PrimBool;
+static VOID: Type = Type::Void;
 static NEVER: Type = Type::Never;
 static GENERICT: Type = Type::GenericParam("T");
 static GENERICU: Type = Type::GenericParam("U");
@@ -151,19 +153,70 @@ static U8X8X2: Type = Type::U(8, 8, 2);
 static U8X8X3: Type = Type::U(8, 8, 3);
 static U8X8X4: Type = Type::U(8, 8, 4);
 
+static SVBOOL: Type = Type::Pred;
+static SVF32: Type = Type::SVF(32, 1);
+static SVF32X2: Type = Type::SVF(32, 2);
+static SVF32X3: Type = Type::SVF(32, 3);
+static SVF32X4: Type = Type::SVF(32, 4);
+static SVF64: Type = Type::SVF(64, 1);
+static SVF64X2: Type = Type::SVF(64, 2);
+static SVF64X3: Type = Type::SVF(64, 3);
+static SVF64X4: Type = Type::SVF(64, 4);
+static SVI8: Type = Type::SVI(8, 1);
+static SVI8X2: Type = Type::SVI(8, 2);
+static SVI8X3: Type = Type::SVI(8, 3);
+static SVI8X4: Type = Type::SVI(8, 4);
+static SVI16: Type = Type::SVI(16, 1);
+static SVI16X2: Type = Type::SVI(16, 2);
+static SVI16X3: Type = Type::SVI(16, 3);
+static SVI16X4: Type = Type::SVI(16, 4);
+static SVI32: Type = Type::SVI(32, 1);
+static SVI32X2: Type = Type::SVI(32, 2);
+static SVI32X3: Type = Type::SVI(32, 3);
+static SVI32X4: Type = Type::SVI(32, 4);
+static SVI64: Type = Type::SVI(64, 1);
+static SVI64X2: Type = Type::SVI(64, 2);
+static SVI64X3: Type = Type::SVI(64, 3);
+static SVI64X4: Type = Type::SVI(64, 4);
+static SVU8: Type = Type::SVU(8, 1);
+static SVU8X2: Type = Type::SVU(8, 2);
+static SVU8X3: Type = Type::SVU(8, 3);
+static SVU8X4: Type = Type::SVU(8, 4);
+static SVU16: Type = Type::SVU(16, 1);
+static SVU16X2: Type = Type::SVU(16, 2);
+static SVU16X3: Type = Type::SVU(16, 3);
+static SVU16X4: Type = Type::SVU(16, 4);
+static SVU32: Type = Type::SVU(32, 1);
+static SVU32X2: Type = Type::SVU(32, 2);
+static SVU32X3: Type = Type::SVU(32, 3);
+static SVU32X4: Type = Type::SVU(32, 4);
+static SVU64: Type = Type::SVU(64, 1);
+static SVU64X2: Type = Type::SVU(64, 2);
+static SVU64X3: Type = Type::SVU(64, 3);
+static SVU64X4: Type = Type::SVU(64, 4);
+static SVPRFOP: Type = Type::Enum("svprfop");
+static SVPATTERN: Type = Type::Enum("svpattern");
+
 #[derive(Debug, Copy, Clone, PartialEq)]
 enum Type {
+    Void,
+    PrimBool,
     PrimFloat(u8),
     PrimSigned(u8),
     PrimUnsigned(u8),
     PrimPoly(u8),
     MutPtr(&'static Type),
     ConstPtr(&'static Type),
+    Enum(&'static str),
     GenericParam(&'static str),
     I(u8, u8, u8),
     U(u8, u8, u8),
     P(u8, u8, u8),
     F(u8, u8, u8),
+    Pred,
+    SVI(u8, u8),
+    SVU(u8, u8),
+    SVF(u8, u8),
     Never,
 }
 
@@ -182,6 +235,7 @@ fn verify_all_signatures() {
 
     let mut all_valid = true;
     for rust in FUNCTIONS {
+        // Most SVE intrinsics just rely on the intrinsics test tool for validation
         if !rust.has_test {
             let skip = [
                 "vaddq_s64",
@@ -407,18 +461,16 @@ fn verify_all_signatures() {
                 "__clrex",
                 "__dbg",
             ];
-            if !skip.contains(&rust.name) {
+            if !skip.contains(&rust.name)
-                println!(
+                // Most run-time tests are handled by the intrinsic-test tool, except for
-                    "missing run-time test named `test_{}` for `{}`",
+                // load/stores (which have generated tests)
-                    {
+                && (!rust.name.starts_with("sv") || rust.name.starts_with("svld")
-                        let mut id = rust.name;
+                    || rust.name.starts_with("svst"))
-                        while id.starts_with('_') {
+                // The load/store test generator can't handle these cases yet
-                            id = &id[1..];
+                && (!rust.name.contains("_u32base_") || rust.name.contains("index") || rust.name.contains("offset"))
-                        }
+                && !(rust.name.starts_with("svldff1") && rust.name.contains("gather"))
-                        id
+            {
-                    },
+                println!("missing run-time test for `{}`", rust.name);
-                    rust.name
-                );
                 all_valid = false;
             }
         }
@@ -493,12 +545,21 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> {
     let mut nconst = 0;
     let iter = rust.arguments.iter().zip(&arm.arguments).enumerate();
     for (i, (rust_ty, (arm, arm_const))) in iter {
-        if *rust_ty != arm {
+        match (*rust_ty, arm) {
-            bail!("mismatched arguments: {rust_ty:?} != {arm:?}")
+            // SVE uses generic type parameters to handle void pointers
+            (Type::ConstPtr(Type::GenericParam("T")), Type::ConstPtr(Type::Void)) => (),
+            // SVE const generics use i32 over u64 for usability reasons
+            (Type::PrimSigned(32), Type::PrimUnsigned(64)) if rust.required_const.contains(&i) => {
+                ()
+            }
+            // svset doesn't have its const argument last as we assumed when building the Function
+            _ if rust.name.starts_with("svset") => (),
+            (x, y) if x == y => (),
+            _ => bail!("mismatched arguments: {rust_ty:?} != {arm:?}"),
         }
         if *arm_const {
             nconst += 1;
-            if !rust.required_const.contains(&i) {
+            if !rust.required_const.contains(&i) && !rust.name.starts_with("svset") {
                 bail!("argument const mismatch");
             }
         }
@@ -507,7 +568,7 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> {
         bail!("wrong number of const arguments");
     }
 
-    if rust.instrs.is_empty() {
+    if rust.instrs.is_empty() && arm.instruction != "" {
         bail!(
             "instruction not listed for `{}`, but arm lists {:?}",
             rust.name,
@@ -546,7 +607,7 @@ fn matches(rust: &Function, arm: &Intrinsic) -> Result<(), String> {
     Ok(())
 }
 
-#[derive(PartialEq)]
+#[derive(Debug, PartialEq)]
 struct Intrinsic {
     name: String,
     ret: Option<Type>,
@@ -561,7 +622,7 @@ struct JsonIntrinsic {
     arguments: Vec<String>,
     return_type: ReturnType,
     #[serde(default)]
-    instructions: Vec<Vec<String>>,
+    instructions: Option<Vec<Vec<String>>>,
 }
 
 #[derive(Deserialize, Debug)]
@@ -578,8 +639,8 @@ fn parse_intrinsics(intrinsics: Vec<JsonIntrinsic>) -> HashMap<String, Intrinsic
     ret
 }
 
-fn parse_intrinsic(mut intr: JsonIntrinsic) -> Intrinsic {
+fn parse_intrinsic(intr: JsonIntrinsic) -> Intrinsic {
-    let name = intr.name;
+    let name = intr.name.replace('[', "").replace(']', "");
     let ret = if intr.return_type.value == "void" {
         None
     } else {
@@ -588,18 +649,24 @@ fn parse_intrinsic(mut intr: JsonIntrinsic) -> Intrinsic {
 
     // This ignores multiple instructions and different optional sequences for now to mimic
     // the old HTML scraping behaviour
-    let instruction = intr.instructions.swap_remove(0).swap_remove(0);
+    let instruction = intr
+        .instructions
+        .map_or(String::new(), |mut i| i.swap_remove(0).swap_remove(0));
 
     let arguments = intr
         .arguments
         .iter()
         .map(|s| {
-            let (ty, konst) = match s.strip_prefix("const") {
+            let ty = if let Some(i) = s.find('*') {
-                Some(stripped) => (stripped.trim_start(), true),
+                &s[..i + 1]
-                None => (s.as_str(), false),
+            } else {
+                s.rsplit_once(' ').unwrap().0.trim_start_matches("const ")
             };
-            let ty = ty.rsplit_once(' ').unwrap().0;
+            let ty = parse_ty(ty);
-            (parse_ty(ty), konst)
+            let konst = s.contains("const") && !matches!(ty, Type::ConstPtr(_))
+                || s.starts_with("enum")
+                || s.rsplit_once(" ").unwrap().1.starts_with("imm");
+            (ty, konst)
         })
         .collect::<Vec<_>>();
 
@@ -612,18 +679,26 @@ fn parse_intrinsic(mut intr: JsonIntrinsic) -> Intrinsic {
 }
 
 fn parse_ty(s: &str) -> Type {
-    let suffix = " const *";
+    if let Some(ty) = s.strip_suffix("*") {
-    if let Some(base) = s.strip_suffix(suffix) {
+        let ty = ty.trim();
-        Type::ConstPtr(parse_ty_base(base))
+        if let Some(ty) = ty.strip_prefix("const") {
-    } else if let Some(base) = s.strip_suffix(" *") {
+            // SVE intrinsics are west-const (const int8_t *)
-        Type::MutPtr(parse_ty_base(base))
+            Type::ConstPtr(parse_ty_base(ty))
+        } else if let Some(ty) = ty.strip_suffix("const") {
+            // Neon intrinsics are east-const (int8_t const *)
+            Type::ConstPtr(parse_ty_base(ty))
+        } else {
+            Type::MutPtr(parse_ty_base(ty))
+        }
     } else {
         *parse_ty_base(s)
     }
 }
 
 fn parse_ty_base(s: &str) -> &'static Type {
-    match s {
+    match s.trim() {
+        "bool" => &BOOL,
+        "void" => &VOID,
         "float16_t" => &F16,
         "float16x4_t" => &F16X4,
         "float16x4x2_t" => &F16X4X2,
@@ -753,6 +828,49 @@ fn parse_ty_base(s: &str) -> &'static Type {
         "uint8x8x2_t" => &U8X8X2,
         "uint8x8x3_t" => &U8X8X3,
         "uint8x8x4_t" => &U8X8X4,
+        "svbool_t" => &SVBOOL,
+        "svfloat32_t" => &SVF32,
+        "svfloat32x2_t" => &SVF32X2,
+        "svfloat32x3_t" => &SVF32X3,
+        "svfloat32x4_t" => &SVF32X4,
+        "svfloat64_t" => &SVF64,
+        "svfloat64x2_t" => &SVF64X2,
+        "svfloat64x3_t" => &SVF64X3,
+        "svfloat64x4_t" => &SVF64X4,
+        "svint8_t" => &SVI8,
+        "svint8x2_t" => &SVI8X2,
+        "svint8x3_t" => &SVI8X3,
+        "svint8x4_t" => &SVI8X4,
+        "svint16_t" => &SVI16,
+        "svint16x2_t" => &SVI16X2,
+        "svint16x3_t" => &SVI16X3,
+        "svint16x4_t" => &SVI16X4,
+        "svint32_t" => &SVI32,
+        "svint32x2_t" => &SVI32X2,
+        "svint32x3_t" => &SVI32X3,
+        "svint32x4_t" => &SVI32X4,
+        "svint64_t" => &SVI64,
+        "svint64x2_t" => &SVI64X2,
+        "svint64x3_t" => &SVI64X3,
+        "svint64x4_t" => &SVI64X4,
+        "svuint8_t" => &SVU8,
+        "svuint8x2_t" => &SVU8X2,
+        "svuint8x3_t" => &SVU8X3,
+        "svuint8x4_t" => &SVU8X4,
+        "svuint16_t" => &SVU16,
+        "svuint16x2_t" => &SVU16X2,
+        "svuint16x3_t" => &SVU16X3,
+        "svuint16x4_t" => &SVU16X4,
+        "svuint32_t" => &SVU32,
+        "svuint32x2_t" => &SVU32X2,
+        "svuint32x3_t" => &SVU32X3,
+        "svuint32x4_t" => &SVU32X4,
+        "svuint64_t" => &SVU64,
+        "svuint64x2_t" => &SVU64X2,
+        "svuint64x3_t" => &SVU64X3,
+        "svuint64x4_t" => &SVU64X4,
+        "enum svprfop" => &SVPRFOP,
+        "enum svpattern" => &SVPATTERN,
 
         _ => panic!("failed to parse json type {s:?}"),
     }

library/stdarch/vendor.yml

@@ -1,2 +1,3 @@
-- crates/stdarch-verify/arm-intrinsics.html
 - crates/stdarch-verify/x86-intel.xml
+- crates/stdarch-verify/mips-msa.h
+- intrinsics_data/arm_intrinsics.json