1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines all of the AARCH64-specific intrinsics. 10// 11//===----------------------------------------------------------------------===// 12 13let TargetPrefix = "aarch64" in { 14 15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty], 16 [IntrNoFree, IntrWillReturn]>; 17def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty], 18 [IntrNoFree, IntrWillReturn]>; 19def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty], 20 [IntrNoFree, IntrWillReturn]>; 21def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty], 22 [IntrNoFree, IntrWillReturn]>; 23 24def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty], 25 [IntrNoFree, IntrWillReturn]>; 26def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty], 27 [IntrNoFree, IntrWillReturn]>; 28def int_aarch64_stxp : Intrinsic<[llvm_i32_ty], 29 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], 30 [IntrNoFree, IntrWillReturn]>; 31def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty], 32 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], 33 [IntrNoFree, IntrWillReturn]>; 34 35def int_aarch64_clrex : Intrinsic<[]>; 36 37def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 38 LLVMMatchType<0>], [IntrNoMem]>; 39def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 40 LLVMMatchType<0>], [IntrNoMem]>; 41 42def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; 43 44def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 45def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; 46 47def int_aarch64_frint32z 48 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 49 [ IntrNoMem ]>; 50def int_aarch64_frint64z 51 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 52 [ IntrNoMem ]>; 53def int_aarch64_frint32x 54 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 55 [ IntrNoMem ]>; 56def int_aarch64_frint64x 57 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 58 [ IntrNoMem ]>; 59 60 61//===----------------------------------------------------------------------===// 62// HINT 63 64def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>; 65 66def int_aarch64_break : Intrinsic<[], [llvm_i32_ty], 67 [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>; 68 69def int_aarch64_hlt : Intrinsic<[], [llvm_i32_ty], 70 [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>; 71 72def int_aarch64_prefetch : Intrinsic<[], 73 [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], 74 [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, ReadOnly<ArgIndex<0>>, 75 ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>> 76 ]>, 77 ClangBuiltin<"__builtin_arm_prefetch">; 78 79//===----------------------------------------------------------------------===// 80// Data Barrier Instructions 81 82def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, 83 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 84def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, 85 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 86def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, 87 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 88 89// A space-consuming intrinsic primarily for testing block and jump table 90// placements. The first argument is the number of bytes this "instruction" 91// takes up, the second and return value are essentially chains, used to force 92// ordering during ISel. 93def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>; 94 95//===----------------------------------------------------------------------===// 96// Guarded Control Stack 97 98def int_aarch64_chkfeat : ClangBuiltin<"__builtin_arm_chkfeat">, 99 DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty], 100 [IntrNoMem]>; 101 102// FIXME: This should be marked as [IntrReadMem, IntrHasSideEffects], as it has 103// the side-effect of updating gcspr, but this combination doesn't work 104// correctly. 105def int_aarch64_gcspopm : ClangBuiltin<"__builtin_arm_gcspopm">, 106 DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty], 107 []>; 108 109def int_aarch64_gcsss : ClangBuiltin<"__builtin_arm_gcsss">, 110 DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty], []>; 111 112} 113 114//===----------------------------------------------------------------------===// 115// Advanced SIMD (NEON) 116 117let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 118 class AdvSIMD_2Scalar_Float_Intrinsic 119 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 120 [IntrNoMem]>; 121 122 class AdvSIMD_FPToIntRounding_Intrinsic 123 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>; 124 125 class AdvSIMD_1IntArg_Intrinsic 126 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>; 127 class AdvSIMD_1FloatArg_Intrinsic 128 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; 129 class AdvSIMD_1VectorArg_Intrinsic 130 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>; 131 class AdvSIMD_1VectorArg_Expand_Intrinsic 132 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>; 133 class AdvSIMD_1VectorArg_Long_Intrinsic 134 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>; 135 class AdvSIMD_1IntArg_Narrow_Intrinsic 136 : DefaultAttrsIntrinsic<[llvm_any_ty], [llvm_any_ty], [IntrNoMem]>; 137 class AdvSIMD_1VectorArg_Narrow_Intrinsic 138 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>; 139 class AdvSIMD_1VectorArg_Int_Across_Intrinsic 140 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>; 141 class AdvSIMD_1VectorArg_Float_Across_Intrinsic 142 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>; 143 144 class AdvSIMD_2IntArg_Intrinsic 145 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 146 [IntrNoMem]>; 147 class AdvSIMD_2FloatArg_Intrinsic 148 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 149 [IntrNoMem]>; 150 class AdvSIMD_2VectorArg_Intrinsic 151 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 152 [IntrNoMem]>; 153 class AdvSIMD_2VectorArg_Compare_Intrinsic 154 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>], 155 [IntrNoMem]>; 156 class AdvSIMD_2Arg_FloatCompare_Intrinsic 157 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>], 158 [IntrNoMem]>; 159 class AdvSIMD_2VectorArg_Long_Intrinsic 160 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 161 [LLVMTruncatedType<0>, LLVMTruncatedType<0>], 162 [IntrNoMem]>; 163 class AdvSIMD_2VectorArg_Wide_Intrinsic 164 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 165 [LLVMMatchType<0>, LLVMTruncatedType<0>], 166 [IntrNoMem]>; 167 class AdvSIMD_2VectorArg_Narrow_Intrinsic 168 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 169 [LLVMExtendedType<0>, LLVMExtendedType<0>], 170 [IntrNoMem]>; 171 class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic 172 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 173 [LLVMExtendedType<0>, llvm_i32_ty], 174 [IntrNoMem]>; 175 class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic 176 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 177 [llvm_anyvector_ty], 178 [IntrNoMem]>; 179 class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic 180 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 181 [LLVMTruncatedType<0>], 182 [IntrNoMem]>; 183 class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic 184 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 185 [LLVMTruncatedType<0>, llvm_i32_ty], 186 [IntrNoMem]>; 187 class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic 188 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 189 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty], 190 [IntrNoMem]>; 191 class AdvSIMD_2VectorArg_Lane_Intrinsic 192 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 193 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty], 194 [IntrNoMem]>; 195 196 class AdvSIMD_3IntArg_Intrinsic 197 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 198 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 199 [IntrNoMem]>; 200 class AdvSIMD_3VectorArg_Intrinsic 201 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 202 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 203 [IntrNoMem]>; 204 class AdvSIMD_3VectorArg_Scalar_Intrinsic 205 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 206 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], 207 [IntrNoMem]>; 208 class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic 209 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 210 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, 211 LLVMMatchType<1>], [IntrNoMem]>; 212 class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic 213 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 214 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty], 215 [IntrNoMem]>; 216 class AdvSIMD_CvtFxToFP_Intrinsic 217 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], 218 [IntrNoMem]>; 219 class AdvSIMD_CvtFPToFx_Intrinsic 220 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty], 221 [IntrNoMem]>; 222 223 class AdvSIMD_1Arg_Intrinsic 224 : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>; 225 226 class AdvSIMD_Dot_Intrinsic 227 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 228 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 229 [IntrNoMem]>; 230 231 class AdvSIMD_FP16FML_Intrinsic 232 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 233 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 234 [IntrNoMem]>; 235 236 class AdvSIMD_MatMul_Intrinsic 237 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 238 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 239 [IntrNoMem]>; 240 241 class AdvSIMD_FML_Intrinsic 242 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 243 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 244 [IntrNoMem]>; 245 246 class AdvSIMD_BF16FML_Intrinsic 247 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], 248 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 249 [IntrNoMem]>; 250} 251 252// Arithmetic ops 253 254let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { 255 // Vector Add Across Lanes 256 def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 257 def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 258 def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 259 260 // Vector Long Add Across Lanes 261 def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 262 def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 263 264 // Vector Halving Add 265 def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic; 266 def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic; 267 268 // Vector Rounding Halving Add 269 def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic; 270 def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic; 271 272 // Vector Saturating Add 273 def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic; 274 def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic; 275 def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic; 276 def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic; 277 278 // Vector Add High-Half 279 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 280 // header is no longer supported. 281 def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 282 283 // Vector Rounding Add High-Half 284 def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 285 286 // Vector Saturating Doubling Multiply High 287 def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic; 288 def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 289 def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 290 291 // Vector Saturating Rounding Doubling Multiply High 292 def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic; 293 def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 294 def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 295 296 def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic; 297 def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic; 298 299 // Vector Polynominal Multiply 300 def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic; 301 302 // Vector Long Multiply 303 def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic; 304 def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic; 305 def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic; 306 307 // 64-bit polynomial multiply really returns an i128, which is not legal. Fake 308 // it with a v16i8. 309 def int_aarch64_neon_pmull64 : 310 DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 311 312 // Vector Extending Multiply 313 def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic { 314 let IntrProperties = [IntrNoMem, Commutative]; 315 } 316 317 // Vector Saturating Doubling Long Multiply 318 def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic; 319 def int_aarch64_neon_sqdmulls_scalar 320 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; 321 322 // Vector Halving Subtract 323 def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic; 324 def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic; 325 326 // Vector Saturating Subtract 327 def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic; 328 def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic; 329 330 // Vector Subtract High-Half 331 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 332 // header is no longer supported. 333 def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 334 335 // Vector Rounding Subtract High-Half 336 def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 337 338 // Vector Compare Absolute Greater-than-or-equal 339 def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic; 340 341 // Vector Compare Absolute Greater-than 342 def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic; 343 344 // Vector Absolute Difference 345 def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic; 346 def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic; 347 def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic; 348 349 // Scalar Absolute Difference 350 def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic; 351 352 // Vector Max 353 def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic; 354 def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic; 355 def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic; 356 def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic; 357 358 // Vector Max Across Lanes 359 def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 360 def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 361 def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 362 def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 363 364 // Vector Min 365 def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic; 366 def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic; 367 def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic; 368 def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic; 369 370 // Vector Min/Max Number 371 def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic; 372 def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic; 373 374 // Vector Min Across Lanes 375 def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 376 def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 377 def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 378 def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 379 380 // Pairwise Add 381 def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic; 382 def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic; 383 384 // Long Pairwise Add 385 // FIXME: In theory, we shouldn't need intrinsics for saddlp or 386 // uaddlp, but tblgen's type inference currently can't handle the 387 // pattern fragments this ends up generating. 388 def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 389 def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 390 391 // Folding Maximum 392 def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic; 393 def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic; 394 def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic; 395 396 // Folding Minimum 397 def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic; 398 def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic; 399 def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic; 400 401 // Reciprocal Estimate/Step 402 def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic; 403 def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic; 404 405 // Reciprocal Exponent 406 def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic; 407 408 // Vector Saturating Shift Left 409 def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic; 410 def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic; 411 412 // Vector Rounding Shift Left 413 def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic; 414 def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic; 415 416 // Vector Saturating Rounding Shift Left 417 def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic; 418 def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic; 419 420 // Vector Signed->Unsigned Shift Left by Constant 421 def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic; 422 423 // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant 424 def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 425 426 // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const 427 def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 428 429 // Vector Narrowing Shift Right by Constant 430 def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 431 def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 432 433 // Vector Rounding Narrowing Shift Right by Constant 434 def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 435 436 // Vector Rounding Narrowing Saturating Shift Right by Constant 437 def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 438 def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 439 440 // Vector Shift Left 441 def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic; 442 def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic; 443 444 // Vector Widening Shift Left by Constant 445 def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic; 446 def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 447 def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 448 449 // Vector Shift Right by Constant and Insert 450 def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic; 451 452 // Vector Shift Left by Constant and Insert 453 def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic; 454 455 // Vector Saturating Narrow 456 def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic; 457 def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic; 458 def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 459 def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 460 461 // Vector Saturating Extract and Unsigned Narrow 462 def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic; 463 def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic; 464 465 // Vector Absolute Value 466 def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic; 467 468 // Vector Saturating Absolute Value 469 def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic; 470 471 // Vector Saturating Negation 472 def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic; 473 474 // Vector Count Leading Sign Bits 475 def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic; 476 477 // Vector Reciprocal Estimate 478 def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic; 479 def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic; 480 481 // Vector Square Root Estimate 482 def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic; 483 def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic; 484 485 // Vector Conversions Between Half-Precision and Single-Precision. 486 def int_aarch64_neon_vcvtfp2hf 487 : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 488 def int_aarch64_neon_vcvthf2fp 489 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>; 490 491 // Vector Conversions Between Floating-point and Fixed-point. 492 def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic; 493 def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic; 494 def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic; 495 def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic; 496 497 // Vector FP->Int Conversions 498 def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic; 499 def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic; 500 def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic; 501 def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic; 502 def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic; 503 def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic; 504 def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic; 505 def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic; 506 def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic; 507 def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic; 508 509 // v8.5-A Vector FP Rounding 510 def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic; 511 def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic; 512 def int_aarch64_neon_frint64x : AdvSIMD_1FloatArg_Intrinsic; 513 def int_aarch64_neon_frint64z : AdvSIMD_1FloatArg_Intrinsic; 514 515 // Scalar FP->Int conversions 516 517 // Vector FP Inexact Narrowing 518 def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic; 519 520 // Scalar FP Inexact Narrowing 521 def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], 522 [IntrNoMem]>; 523 524 // v8.2-A Dot Product 525 def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic; 526 def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic; 527 528 // v8.6-A Matrix Multiply Intrinsics 529 def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic; 530 def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic; 531 def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic; 532 def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic; 533 def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic; 534 def int_aarch64_neon_bfmmla 535 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], 536 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 537 [IntrNoMem]>; 538 def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic; 539 def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic; 540 541 542 // v8.6-A Bfloat Intrinsics 543 def int_aarch64_neon_bfcvt 544 : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>; 545 def int_aarch64_neon_bfcvtn 546 : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 547 def int_aarch64_neon_bfcvtn2 548 : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], 549 [llvm_v8bf16_ty, llvm_v4f32_ty], 550 [IntrNoMem]>; 551 552 // v8.2-A FP16 Fused Multiply-Add Long 553 def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic; 554 def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic; 555 def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic; 556 def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic; 557 558 // v8.3-A Floating-point complex add 559 def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic; 560 def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic; 561 562 def int_aarch64_neon_vcmla_rot0 : AdvSIMD_3VectorArg_Intrinsic; 563 def int_aarch64_neon_vcmla_rot90 : AdvSIMD_3VectorArg_Intrinsic; 564 def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic; 565 def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic; 566} 567 568let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 569 class AdvSIMD_2Vector2Index_Intrinsic 570 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 571 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty], 572 [IntrNoMem]>; 573} 574 575// Vector element to element moves 576def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic; 577 578let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 579 class AdvSIMD_1Vec_Load_Intrinsic 580 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyptr_ty], 581 [IntrReadMem, IntrArgMemOnly]>; 582 class AdvSIMD_1Vec_Store_Lane_Intrinsic 583 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], 584 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 585 586 class AdvSIMD_2Vec_Load_Intrinsic 587 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], 588 [llvm_anyptr_ty], 589 [IntrReadMem, IntrArgMemOnly]>; 590 class AdvSIMD_2Vec_Load_Lane_Intrinsic 591 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>], 592 [LLVMMatchType<0>, llvm_anyvector_ty, 593 llvm_i64_ty, llvm_anyptr_ty], 594 [IntrReadMem, IntrArgMemOnly]>; 595 class AdvSIMD_2Vec_Store_Intrinsic 596 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 597 llvm_anyptr_ty], 598 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 599 class AdvSIMD_2Vec_Store_Lane_Intrinsic 600 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 601 llvm_i64_ty, llvm_anyptr_ty], 602 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 603 604 class AdvSIMD_3Vec_Load_Intrinsic 605 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty], 606 [llvm_anyptr_ty], 607 [IntrReadMem, IntrArgMemOnly]>; 608 class AdvSIMD_3Vec_Load_Lane_Intrinsic 609 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 610 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, 611 llvm_i64_ty, llvm_anyptr_ty], 612 [IntrReadMem, IntrArgMemOnly]>; 613 class AdvSIMD_3Vec_Store_Intrinsic 614 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 615 LLVMMatchType<0>, llvm_anyptr_ty], 616 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 617 class AdvSIMD_3Vec_Store_Lane_Intrinsic 618 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, 619 LLVMMatchType<0>, LLVMMatchType<0>, 620 llvm_i64_ty, llvm_anyptr_ty], 621 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 622 623 class AdvSIMD_4Vec_Load_Intrinsic 624 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 625 LLVMMatchType<0>, llvm_anyvector_ty], 626 [llvm_anyptr_ty], 627 [IntrReadMem, IntrArgMemOnly]>; 628 class AdvSIMD_4Vec_Load_Lane_Intrinsic 629 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 630 LLVMMatchType<0>, LLVMMatchType<0>], 631 [LLVMMatchType<0>, LLVMMatchType<0>, 632 LLVMMatchType<0>, llvm_anyvector_ty, 633 llvm_i64_ty, llvm_anyptr_ty], 634 [IntrReadMem, IntrArgMemOnly]>; 635 class AdvSIMD_4Vec_Store_Intrinsic 636 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 637 LLVMMatchType<0>, LLVMMatchType<0>, 638 llvm_anyptr_ty], 639 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 640 class AdvSIMD_4Vec_Store_Lane_Intrinsic 641 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 642 LLVMMatchType<0>, LLVMMatchType<0>, 643 llvm_i64_ty, llvm_anyptr_ty], 644 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 645} 646 647// Memory ops 648 649def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic; 650def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic; 651def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic; 652 653def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic; 654def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic; 655def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic; 656 657def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic; 658def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic; 659def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic; 660 661def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic; 662def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic; 663def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic; 664 665def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic; 666def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic; 667def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic; 668 669def int_aarch64_neon_st2 : AdvSIMD_2Vec_Store_Intrinsic; 670def int_aarch64_neon_st3 : AdvSIMD_3Vec_Store_Intrinsic; 671def int_aarch64_neon_st4 : AdvSIMD_4Vec_Store_Intrinsic; 672 673def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic; 674def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic; 675def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic; 676 677let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 678 class AdvSIMD_Tbl1_Intrinsic 679 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>], 680 [IntrNoMem]>; 681 class AdvSIMD_Tbl2_Intrinsic 682 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 683 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>; 684 class AdvSIMD_Tbl3_Intrinsic 685 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 686 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 687 LLVMMatchType<0>], 688 [IntrNoMem]>; 689 class AdvSIMD_Tbl4_Intrinsic 690 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 691 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 692 LLVMMatchType<0>], 693 [IntrNoMem]>; 694 695 class AdvSIMD_Tbx1_Intrinsic 696 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 697 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>], 698 [IntrNoMem]>; 699 class AdvSIMD_Tbx2_Intrinsic 700 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 701 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 702 LLVMMatchType<0>], 703 [IntrNoMem]>; 704 class AdvSIMD_Tbx3_Intrinsic 705 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 706 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 707 llvm_v16i8_ty, LLVMMatchType<0>], 708 [IntrNoMem]>; 709 class AdvSIMD_Tbx4_Intrinsic 710 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 711 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 712 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], 713 [IntrNoMem]>; 714} 715def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic; 716def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic; 717def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic; 718def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic; 719 720def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic; 721def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic; 722def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic; 723def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic; 724 725let TargetPrefix = "aarch64" in { 726 class FPENV_Get_Intrinsic 727 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>; 728 class FPENV_Set_Intrinsic 729 : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; 730 class RNDR_Intrinsic 731 : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>; 732} 733 734// FP environment registers. 735def int_aarch64_get_fpcr : FPENV_Get_Intrinsic; 736def int_aarch64_set_fpcr : FPENV_Set_Intrinsic; 737def int_aarch64_get_fpsr : FPENV_Get_Intrinsic; 738def int_aarch64_set_fpsr : FPENV_Set_Intrinsic; 739 740// Armv8.5-A Random number generation intrinsics 741def int_aarch64_rndr : RNDR_Intrinsic; 742def int_aarch64_rndrrs : RNDR_Intrinsic; 743 744let TargetPrefix = "aarch64" in { 745 class Crypto_AES_DataKey_Intrinsic 746 : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; 747 748 class Crypto_AES_Data_Intrinsic 749 : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; 750 751 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 752 // (v4i32). 753 class Crypto_SHA_5Hash4Schedule_Intrinsic 754 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], 755 [IntrNoMem]>; 756 757 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 758 // (v4i32). 759 class Crypto_SHA_1Hash_Intrinsic 760 : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 761 762 // SHA intrinsic taking 8 words of the schedule 763 class Crypto_SHA_8Schedule_Intrinsic 764 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 765 766 // SHA intrinsic taking 12 words of the schedule 767 class Crypto_SHA_12Schedule_Intrinsic 768 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 769 [IntrNoMem]>; 770 771 // SHA intrinsic taking 8 words of the hash and 4 of the schedule. 772 class Crypto_SHA_8Hash4Schedule_Intrinsic 773 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 774 [IntrNoMem]>; 775 776 // SHA512 intrinsic taking 2 arguments 777 class Crypto_SHA512_2Arg_Intrinsic 778 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; 779 780 // SHA512 intrinsic taking 3 Arguments 781 class Crypto_SHA512_3Arg_Intrinsic 782 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], 783 [IntrNoMem]>; 784 785 // SHA3 Intrinsics taking 3 arguments 786 class Crypto_SHA3_3Arg_Intrinsic 787 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 788 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 789 [IntrNoMem]>; 790 791 // SHA3 Intrinsic taking 2 arguments 792 class Crypto_SHA3_2Arg_Intrinsic 793 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], 794 [IntrNoMem]>; 795 796 // SHA3 Intrinsic taking 3 Arguments 1 immediate 797 class Crypto_SHA3_2ArgImm_Intrinsic 798 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i64_ty], 799 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 800 801 class Crypto_SM3_3Vector_Intrinsic 802 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 803 [IntrNoMem]>; 804 805 class Crypto_SM3_3VectorIndexed_Intrinsic 806 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i64_ty], 807 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 808 809 class Crypto_SM4_2Vector_Intrinsic 810 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 811} 812 813// AES 814def int_aarch64_crypto_aese : Crypto_AES_DataKey_Intrinsic; 815def int_aarch64_crypto_aesd : Crypto_AES_DataKey_Intrinsic; 816def int_aarch64_crypto_aesmc : Crypto_AES_Data_Intrinsic; 817def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic; 818 819// SHA1 820def int_aarch64_crypto_sha1c : Crypto_SHA_5Hash4Schedule_Intrinsic; 821def int_aarch64_crypto_sha1p : Crypto_SHA_5Hash4Schedule_Intrinsic; 822def int_aarch64_crypto_sha1m : Crypto_SHA_5Hash4Schedule_Intrinsic; 823def int_aarch64_crypto_sha1h : Crypto_SHA_1Hash_Intrinsic; 824 825def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic; 826def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic; 827 828// SHA256 829def int_aarch64_crypto_sha256h : Crypto_SHA_8Hash4Schedule_Intrinsic; 830def int_aarch64_crypto_sha256h2 : Crypto_SHA_8Hash4Schedule_Intrinsic; 831def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic; 832def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic; 833 834//SHA3 835def int_aarch64_crypto_eor3s : Crypto_SHA3_3Arg_Intrinsic; 836def int_aarch64_crypto_eor3u : Crypto_SHA3_3Arg_Intrinsic; 837def int_aarch64_crypto_bcaxs : Crypto_SHA3_3Arg_Intrinsic; 838def int_aarch64_crypto_bcaxu : Crypto_SHA3_3Arg_Intrinsic; 839def int_aarch64_crypto_rax1 : Crypto_SHA3_2Arg_Intrinsic; 840def int_aarch64_crypto_xar : Crypto_SHA3_2ArgImm_Intrinsic; 841 842// SHA512 843def int_aarch64_crypto_sha512h : Crypto_SHA512_3Arg_Intrinsic; 844def int_aarch64_crypto_sha512h2 : Crypto_SHA512_3Arg_Intrinsic; 845def int_aarch64_crypto_sha512su0 : Crypto_SHA512_2Arg_Intrinsic; 846def int_aarch64_crypto_sha512su1 : Crypto_SHA512_3Arg_Intrinsic; 847 848//SM3 & SM4 849def int_aarch64_crypto_sm3partw1 : Crypto_SM3_3Vector_Intrinsic; 850def int_aarch64_crypto_sm3partw2 : Crypto_SM3_3Vector_Intrinsic; 851def int_aarch64_crypto_sm3ss1 : Crypto_SM3_3Vector_Intrinsic; 852def int_aarch64_crypto_sm3tt1a : Crypto_SM3_3VectorIndexed_Intrinsic; 853def int_aarch64_crypto_sm3tt1b : Crypto_SM3_3VectorIndexed_Intrinsic; 854def int_aarch64_crypto_sm3tt2a : Crypto_SM3_3VectorIndexed_Intrinsic; 855def int_aarch64_crypto_sm3tt2b : Crypto_SM3_3VectorIndexed_Intrinsic; 856def int_aarch64_crypto_sm4e : Crypto_SM4_2Vector_Intrinsic; 857def int_aarch64_crypto_sm4ekey : Crypto_SM4_2Vector_Intrinsic; 858 859//===----------------------------------------------------------------------===// 860// CRC32 861 862let TargetPrefix = "aarch64" in { 863 864def int_aarch64_crc32b : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 865 [IntrNoMem]>; 866def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 867 [IntrNoMem]>; 868def int_aarch64_crc32h : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 869 [IntrNoMem]>; 870def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 871 [IntrNoMem]>; 872def int_aarch64_crc32w : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 873 [IntrNoMem]>; 874def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 875 [IntrNoMem]>; 876def int_aarch64_crc32x : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 877 [IntrNoMem]>; 878def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 879 [IntrNoMem]>; 880} 881 882//===----------------------------------------------------------------------===// 883// Memory Tagging Extensions (MTE) Intrinsics 884let TargetPrefix = "aarch64" in { 885def int_aarch64_irg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 886 [IntrNoMem, IntrHasSideEffects]>; 887def int_aarch64_addg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 888 [IntrNoMem]>; 889def int_aarch64_gmi : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], 890 [IntrNoMem]>; 891def int_aarch64_ldg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], 892 [IntrReadMem]>; 893def int_aarch64_stg : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], 894 [IntrWriteMem]>; 895def int_aarch64_subp : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], 896 [IntrNoMem]>; 897 898// The following are codegen-only intrinsics for stack instrumentation. 899 900// Generate a randomly tagged stack base pointer. 901def int_aarch64_irg_sp : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty], 902 [IntrNoMem, IntrHasSideEffects]>; 903 904// Transfer pointer tag with offset. 905// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where 906// * address is the address in ptr0 907// * tag is a function of (tag in baseptr, tag_offset). 908// ** Beware, this is not the same function as implemented by the ADDG instruction! 909// Backend optimizations may change tag_offset; the only guarantee is that calls 910// to tagp with the same pair of (baseptr, tag_offset) will produce pointers 911// with the same tag value, assuming the set of excluded tags has not changed. 912// Address bits in baseptr and tag bits in ptr0 are ignored. 913// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as 914// ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset 915// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp. 916def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty], 917 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 918 919// Update allocation tags for the memory range to match the tag in the pointer argument. 920def int_aarch64_settag : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 921 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 922 923// Update allocation tags for the memory range to match the tag in the pointer argument, 924// and set memory contents to zero. 925def int_aarch64_settag_zero : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 926 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 927 928// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values. 929def int_aarch64_stgp : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], 930 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 931} 932 933//===----------------------------------------------------------------------===// 934// Memory Operations (MOPS) Intrinsics 935let TargetPrefix = "aarch64" in { 936 // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64 937 def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty], 938 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 939} 940 941// Transactional Memory Extension (TME) Intrinsics 942let TargetPrefix = "aarch64" in { 943def int_aarch64_tstart : ClangBuiltin<"__builtin_arm_tstart">, 944 Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>; 945 946def int_aarch64_tcommit : ClangBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>; 947 948def int_aarch64_tcancel : ClangBuiltin<"__builtin_arm_tcancel">, 949 Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>; 950 951def int_aarch64_ttest : ClangBuiltin<"__builtin_arm_ttest">, 952 Intrinsic<[llvm_i64_ty], [], 953 [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; 954 955// Armv8.7-A load/store 64-byte intrinsics 956defvar data512 = !listsplat(llvm_i64_ty, 8); 957def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>; 958def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>; 959def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; 960def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; 961 962} 963 964def llvm_nxv1i1_ty : LLVMType<nxv1i1>; 965def llvm_nxv2i1_ty : LLVMType<nxv2i1>; 966def llvm_nxv4i1_ty : LLVMType<nxv4i1>; 967def llvm_nxv8i1_ty : LLVMType<nxv8i1>; 968def llvm_nxv16i1_ty : LLVMType<nxv16i1>; 969def llvm_nxv16i8_ty : LLVMType<nxv16i8>; 970def llvm_nxv4i32_ty : LLVMType<nxv4i32>; 971def llvm_nxv2i64_ty : LLVMType<nxv2i64>; 972def llvm_nxv8f16_ty : LLVMType<nxv8f16>; 973def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>; 974def llvm_nxv4f32_ty : LLVMType<nxv4f32>; 975def llvm_nxv2f64_ty : LLVMType<nxv2f64>; 976 977let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 978 979 class AdvSIMD_1Vec_PredLoad_Intrinsic 980 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 981 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 982 [IntrReadMem, IntrArgMemOnly]>; 983 984 class AdvSIMD_2Vec_PredLoad_Intrinsic 985 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 986 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 987 [IntrReadMem, IntrArgMemOnly]>; 988 989 class AdvSIMD_3Vec_PredLoad_Intrinsic 990 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], 991 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 992 [IntrReadMem, IntrArgMemOnly]>; 993 994 class AdvSIMD_4Vec_PredLoad_Intrinsic 995 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 996 LLVMMatchType<0>], 997 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 998 [IntrReadMem, IntrArgMemOnly]>; 999 1000 class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic 1001 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1002 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1003 [IntrInaccessibleMemOrArgMemOnly]>; 1004 1005 class AdvSIMD_1Vec_PredStore_Intrinsic 1006 : DefaultAttrsIntrinsic<[], 1007 [llvm_anyvector_ty, 1008 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1009 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 1010 1011 class AdvSIMD_2Vec_PredStore_Intrinsic 1012 : DefaultAttrsIntrinsic<[], 1013 [llvm_anyvector_ty, LLVMMatchType<0>, 1014 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1015 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 1016 1017 class AdvSIMD_3Vec_PredStore_Intrinsic 1018 : DefaultAttrsIntrinsic<[], 1019 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 1020 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1021 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 1022 1023 class AdvSIMD_4Vec_PredStore_Intrinsic 1024 : DefaultAttrsIntrinsic<[], 1025 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 1026 LLVMMatchType<0>, 1027 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1028 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 1029 1030 class AdvSIMD_SVE_Index_Intrinsic 1031 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1032 [LLVMVectorElementType<0>, 1033 LLVMVectorElementType<0>], 1034 [IntrNoMem]>; 1035 1036 class AdvSIMD_Merged1VectorArg_Intrinsic 1037 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1038 [LLVMMatchType<0>, 1039 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1040 LLVMMatchType<0>], 1041 [IntrNoMem]>; 1042 1043 class AdvSIMD_2VectorArgIndexed_Intrinsic 1044 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1045 [LLVMMatchType<0>, 1046 LLVMMatchType<0>, 1047 llvm_i32_ty], 1048 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1049 1050 class AdvSIMD_3VectorArgIndexed_Intrinsic 1051 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1052 [LLVMMatchType<0>, 1053 LLVMMatchType<0>, 1054 LLVMMatchType<0>, 1055 llvm_i32_ty], 1056 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1057 1058 class AdvSIMD_Pred1VectorArg_Intrinsic 1059 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1060 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1061 LLVMMatchType<0>], 1062 [IntrNoMem]>; 1063 1064 class AdvSIMD_Pred2VectorArg_Intrinsic 1065 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1066 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1067 LLVMMatchType<0>, 1068 LLVMMatchType<0>], 1069 [IntrNoMem]>; 1070 1071 class AdvSIMD_Pred3VectorArg_Intrinsic 1072 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1073 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1074 LLVMMatchType<0>, 1075 LLVMMatchType<0>, 1076 LLVMMatchType<0>], 1077 [IntrNoMem]>; 1078 1079 class AdvSIMD_SVE_Compare_Intrinsic 1080 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1081 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1082 llvm_anyvector_ty, 1083 LLVMMatchType<0>], 1084 [IntrNoMem]>; 1085 1086 class AdvSIMD_SVE_CompareWide_Intrinsic 1087 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1088 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1089 llvm_anyvector_ty, 1090 llvm_nxv2i64_ty], 1091 [IntrNoMem]>; 1092 1093 class AdvSIMD_SVE_Saturating_Intrinsic 1094 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1095 [LLVMMatchType<0>, 1096 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1097 [IntrNoMem]>; 1098 1099 class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic 1100 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1101 [LLVMMatchType<0>, 1102 llvm_i32_ty, 1103 llvm_i32_ty], 1104 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 1105 1106 class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T> 1107 : DefaultAttrsIntrinsic<[T], 1108 [T, llvm_anyvector_ty], 1109 [IntrNoMem]>; 1110 1111 class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T> 1112 : DefaultAttrsIntrinsic<[T], 1113 [T, llvm_i32_ty, llvm_i32_ty], 1114 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 1115 1116 class AdvSIMD_SVE_CNT_Intrinsic 1117 : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>], 1118 [LLVMVectorOfBitcastsToInt<0>, 1119 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1120 llvm_anyvector_ty], 1121 [IntrNoMem]>; 1122 1123 class AdvSIMD_SVE_ReduceWithInit_Intrinsic 1124 : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], 1125 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1126 LLVMVectorElementType<0>, 1127 llvm_anyvector_ty], 1128 [IntrNoMem]>; 1129 1130 class AdvSIMD_SVE_ShiftByImm_Intrinsic 1131 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1132 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1133 LLVMMatchType<0>, 1134 llvm_i32_ty], 1135 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1136 1137 class AdvSIMD_SVE_ShiftWide_Intrinsic 1138 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1139 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1140 LLVMMatchType<0>, 1141 llvm_nxv2i64_ty], 1142 [IntrNoMem]>; 1143 1144 class AdvSIMD_SVE_Unpack_Intrinsic 1145 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1146 [LLVMSubdivide2VectorType<0>], 1147 [IntrNoMem]>; 1148 1149 class AdvSIMD_SVE_CADD_Intrinsic 1150 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1151 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1152 LLVMMatchType<0>, 1153 LLVMMatchType<0>, 1154 llvm_i32_ty], 1155 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1156 1157 class AdvSIMD_SVE_CMLA_Intrinsic 1158 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1159 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1160 LLVMMatchType<0>, 1161 LLVMMatchType<0>, 1162 LLVMMatchType<0>, 1163 llvm_i32_ty], 1164 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 1165 1166 class AdvSIMD_SVE_CMLA_LANE_Intrinsic 1167 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1168 [LLVMMatchType<0>, 1169 LLVMMatchType<0>, 1170 LLVMMatchType<0>, 1171 llvm_i32_ty, 1172 llvm_i32_ty], 1173 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1174 1175 class AdvSIMD_SVE_DUP_Intrinsic 1176 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1177 [LLVMMatchType<0>, 1178 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1179 LLVMVectorElementType<0>], 1180 [IntrNoMem]>; 1181 1182 class AdvSIMD_SVE_DUP_Unpred_Intrinsic 1183 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>], 1184 [IntrNoMem]>; 1185 1186 class AdvSIMD_SVE_DUPQ_Intrinsic 1187 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1188 [LLVMMatchType<0>, 1189 llvm_i64_ty], 1190 [IntrNoMem]>; 1191 1192 class AdvSIMD_SVE_EXPA_Intrinsic 1193 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1194 [LLVMVectorOfBitcastsToInt<0>], 1195 [IntrNoMem]>; 1196 1197 class AdvSIMD_SVE_FCVT_Intrinsic 1198 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1199 [LLVMMatchType<0>, 1200 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1201 llvm_anyvector_ty], 1202 [IntrNoMem]>; 1203 1204 class AdvSIMD_SVE_FCVTZS_Intrinsic 1205 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1206 [LLVMVectorOfBitcastsToInt<0>, 1207 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1208 llvm_anyvector_ty], 1209 [IntrNoMem]>; 1210 1211 class AdvSIMD_SVE_INSR_Intrinsic 1212 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1213 [LLVMMatchType<0>, 1214 LLVMVectorElementType<0>], 1215 [IntrNoMem]>; 1216 1217 class AdvSIMD_SVE_PTRUE_Intrinsic 1218 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1219 [llvm_i32_ty], 1220 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1221 1222 class AdvSIMD_SVE_PUNPKHI_Intrinsic 1223 : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>], 1224 [llvm_anyvector_ty], 1225 [IntrNoMem]>; 1226 1227 class AdvSIMD_SVE_SCALE_Intrinsic 1228 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1229 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1230 LLVMMatchType<0>, 1231 LLVMVectorOfBitcastsToInt<0>], 1232 [IntrNoMem]>; 1233 1234 class AdvSIMD_SVE_SCVTF_Intrinsic 1235 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1236 [LLVMMatchType<0>, 1237 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1238 llvm_anyvector_ty], 1239 [IntrNoMem]>; 1240 1241 class AdvSIMD_SVE_TSMUL_Intrinsic 1242 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1243 [LLVMMatchType<0>, 1244 LLVMVectorOfBitcastsToInt<0>], 1245 [IntrNoMem]>; 1246 1247 class AdvSIMD_SVE_CNTB_Intrinsic 1248 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1249 [llvm_i32_ty], 1250 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1251 1252 class AdvSIMD_SVE_CNTP_Intrinsic 1253 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1254 [llvm_anyvector_ty, LLVMMatchType<0>], 1255 [IntrNoMem]>; 1256 1257 class AdvSIMD_SVE_DOT_Intrinsic 1258 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1259 [LLVMMatchType<0>, 1260 LLVMSubdivide4VectorType<0>, 1261 LLVMSubdivide4VectorType<0>], 1262 [IntrNoMem]>; 1263 1264 class AdvSIMD_SVE_DOT_Indexed_Intrinsic 1265 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1266 [LLVMMatchType<0>, 1267 LLVMSubdivide4VectorType<0>, 1268 LLVMSubdivide4VectorType<0>, 1269 llvm_i32_ty], 1270 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1271 1272 class AdvSIMD_SVE_PTEST_Intrinsic 1273 : DefaultAttrsIntrinsic<[llvm_i1_ty], 1274 [llvm_anyvector_ty, 1275 LLVMMatchType<0>], 1276 [IntrNoMem]>; 1277 1278 class AdvSIMD_SVE_TBL_Intrinsic 1279 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1280 [LLVMMatchType<0>, 1281 LLVMVectorOfBitcastsToInt<0>], 1282 [IntrNoMem]>; 1283 1284 class AdvSIMD_SVE2_TBX_Intrinsic 1285 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1286 [LLVMMatchType<0>, 1287 LLVMMatchType<0>, 1288 LLVMVectorOfBitcastsToInt<0>], 1289 [IntrNoMem]>; 1290 1291 class SVE2_1VectorArg_Long_Intrinsic 1292 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1293 [LLVMSubdivide2VectorType<0>, 1294 llvm_i32_ty], 1295 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1296 1297 class SVE2_2VectorArg_Long_Intrinsic 1298 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1299 [LLVMSubdivide2VectorType<0>, 1300 LLVMSubdivide2VectorType<0>], 1301 [IntrNoMem]>; 1302 1303 class SVE2_2VectorArgIndexed_Long_Intrinsic 1304 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1305 [LLVMSubdivide2VectorType<0>, 1306 LLVMSubdivide2VectorType<0>, 1307 llvm_i32_ty], 1308 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1309 1310 class SVE2_2VectorArg_Wide_Intrinsic 1311 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1312 [LLVMMatchType<0>, 1313 LLVMSubdivide2VectorType<0>], 1314 [IntrNoMem]>; 1315 1316 class SVE2_2VectorArg_Pred_Long_Intrinsic 1317 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1318 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1319 LLVMMatchType<0>, 1320 LLVMSubdivide2VectorType<0>], 1321 [IntrNoMem]>; 1322 1323 class SVE2_3VectorArg_Long_Intrinsic 1324 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1325 [LLVMMatchType<0>, 1326 LLVMSubdivide2VectorType<0>, 1327 LLVMSubdivide2VectorType<0>], 1328 [IntrNoMem]>; 1329 1330 class SVE2_3VectorArgIndexed_Long_Intrinsic 1331 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1332 [LLVMMatchType<0>, 1333 LLVMSubdivide2VectorType<0>, 1334 LLVMSubdivide2VectorType<0>, 1335 llvm_i32_ty], 1336 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1337 1338 class SVE2_1VectorArg_Narrowing_Intrinsic 1339 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1340 [llvm_anyvector_ty], 1341 [IntrNoMem]>; 1342 1343 class SVE2_Merged1VectorArg_Narrowing_Intrinsic 1344 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1345 [LLVMSubdivide2VectorType<0>, 1346 llvm_anyvector_ty], 1347 [IntrNoMem]>; 1348 class SVE2_2VectorArg_Narrowing_Intrinsic 1349 : DefaultAttrsIntrinsic< 1350 [LLVMSubdivide2VectorType<0>], 1351 [llvm_anyvector_ty, LLVMMatchType<0>], 1352 [IntrNoMem]>; 1353 1354 class SVE2_Merged2VectorArg_Narrowing_Intrinsic 1355 : DefaultAttrsIntrinsic< 1356 [LLVMSubdivide2VectorType<0>], 1357 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>], 1358 [IntrNoMem]>; 1359 1360 class SVE2_1VectorArg_Imm_Narrowing_Intrinsic 1361 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1362 [llvm_anyvector_ty, llvm_i32_ty], 1363 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1364 1365 class SVE2_2VectorArg_Imm_Narrowing_Intrinsic 1366 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1367 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, 1368 llvm_i32_ty], 1369 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1370 1371 class SVE2_CONFLICT_DETECT_Intrinsic 1372 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1373 [llvm_anyptr_ty, LLVMMatchType<1>], 1374 [IntrNoMem]>; 1375 1376 class SVE2_3VectorArg_Indexed_Intrinsic 1377 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1378 [LLVMMatchType<0>, 1379 LLVMSubdivide2VectorType<0>, 1380 LLVMSubdivide2VectorType<0>, 1381 llvm_i32_ty], 1382 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1383 1384 class SVE2_1VectorArgIndexed_Intrinsic 1385 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1386 [LLVMMatchType<0>, 1387 llvm_i32_ty], 1388 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1389 1390 class AdvSIMD_SVE_CDOT_LANE_Intrinsic 1391 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1392 [LLVMMatchType<0>, 1393 LLVMSubdivide4VectorType<0>, 1394 LLVMSubdivide4VectorType<0>, 1395 llvm_i32_ty, 1396 llvm_i32_ty], 1397 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1398 1399 class SVE2_1VectorArg_Pred_Intrinsic 1400 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1401 [llvm_anyvector_ty], 1402 [IntrNoMem]>; 1403 1404 class SVE2_1VectorArgIndexed_Pred_Intrinsic 1405 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1406 [llvm_anyvector_ty, llvm_i32_ty], 1407 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1408 1409 class SVE2_Pred_1VectorArgIndexed_Intrinsic 1410 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1411 [LLVMMatchType<0>, 1412 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty], 1413 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1414 1415 class SVE2_Pred_1VectorArg_Intrinsic 1416 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1417 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1418 [IntrNoMem]>; 1419 1420 // NOTE: There is no relationship between these intrinsics beyond an attempt 1421 // to reuse currently identical class definitions. 1422 class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; 1423 class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic; 1424 class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic; 1425 1426 // This class of intrinsics are not intended to be useful within LLVM IR but 1427 // are instead here to support some of the more regid parts of the ACLE. 1428 class Builtin_SVCVT<LLVMType OUT, LLVMType PRED, LLVMType IN> 1429 : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>; 1430} 1431 1432//===----------------------------------------------------------------------===// 1433// SVE 1434 1435let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 1436 1437class AdvSIMD_SVE_2SVBoolArg_Intrinsic 1438 : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 1439 [llvm_nxv16i1_ty], 1440 [IntrNoMem]>; 1441 1442class AdvSIMD_SVE_3SVBoolArg_Intrinsic 1443 : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 1444 [llvm_nxv16i1_ty, llvm_nxv16i1_ty], 1445 [IntrNoMem]>; 1446 1447class AdvSIMD_SVE_Reduce_Intrinsic 1448 : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], 1449 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1450 llvm_anyvector_ty], 1451 [IntrNoMem]>; 1452 1453class AdvSIMD_SVE_V128_Reduce_Intrinsic 1454 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1455 [LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, 1456 llvm_anyvector_ty], 1457 [IntrNoMem]>; 1458 1459 1460class AdvSIMD_SVE_SADDV_Reduce_Intrinsic 1461 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1462 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1463 llvm_anyvector_ty], 1464 [IntrNoMem]>; 1465 1466class AdvSIMD_SVE_WHILE_Intrinsic 1467 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1468 [llvm_anyint_ty, LLVMMatchType<1>], 1469 [IntrNoMem]>; 1470 1471class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic 1472 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1473 [ 1474 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1475 llvm_ptr_ty, 1476 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1477 ], 1478 [IntrReadMem, IntrArgMemOnly]>; 1479 1480class AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic 1481 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1482 [ 1483 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1484 llvm_ptr_ty, 1485 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1486 ], 1487 [IntrInaccessibleMemOrArgMemOnly]>; 1488 1489class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic 1490 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1491 [ 1492 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1493 llvm_ptr_ty, 1494 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1495 ], 1496 [IntrReadMem, IntrArgMemOnly]>; 1497 1498class AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic 1499 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1500 [ 1501 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1502 llvm_ptr_ty, 1503 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1504 ], 1505 [IntrInaccessibleMemOrArgMemOnly]>; 1506 1507class AdvSIMD_GatherLoad_VS_Intrinsic 1508 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1509 [ 1510 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1511 llvm_anyvector_ty, 1512 llvm_i64_ty 1513 ], 1514 [IntrReadMem]>; 1515 1516class AdvSIMD_GatherLoadQ_VS_Intrinsic 1517 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1518 [ 1519 llvm_nxv1i1_ty, 1520 llvm_anyvector_ty, 1521 llvm_i64_ty 1522 ], 1523 [IntrReadMem]>; 1524 1525class AdvSIMD_GatherLoadQ_SV_Intrinsic 1526 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1527 [ 1528 llvm_nxv1i1_ty, 1529 llvm_ptr_ty, 1530 llvm_nxv2i64_ty 1531 ], 1532 [IntrReadMem, IntrArgMemOnly]>; 1533 1534class AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic 1535 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1536 [ 1537 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1538 llvm_anyvector_ty, 1539 llvm_i64_ty 1540 ], 1541 [IntrInaccessibleMemOrArgMemOnly]>; 1542 1543class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic 1544 : DefaultAttrsIntrinsic<[], 1545 [ 1546 llvm_anyvector_ty, 1547 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1548 llvm_ptr_ty, 1549 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1550 ], 1551 [IntrWriteMem, IntrArgMemOnly]>; 1552 1553class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic 1554 : DefaultAttrsIntrinsic<[], 1555 [ 1556 llvm_anyvector_ty, 1557 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1558 llvm_ptr_ty, 1559 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1560 ], 1561 [IntrWriteMem, IntrArgMemOnly]>; 1562 1563class AdvSIMD_ScatterStore_VS_Intrinsic 1564 : DefaultAttrsIntrinsic<[], 1565 [ 1566 llvm_anyvector_ty, 1567 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1568 llvm_anyvector_ty, llvm_i64_ty 1569 ], 1570 [IntrWriteMem]>; 1571 1572class AdvSIMD_ScatterStoreQ_VS_Intrinsic 1573 : DefaultAttrsIntrinsic<[], 1574 [ 1575 llvm_anyvector_ty, 1576 llvm_nxv1i1_ty, 1577 llvm_anyvector_ty, 1578 llvm_i64_ty 1579 ], 1580 [IntrWriteMem]>; 1581 1582class AdvSIMD_ScatterStoreQ_SV_Intrinsic 1583 : DefaultAttrsIntrinsic<[], 1584 [ 1585 llvm_anyvector_ty, 1586 llvm_nxv1i1_ty, 1587 llvm_ptr_ty, 1588 llvm_nxv2i64_ty 1589 ], 1590 [IntrWriteMem, IntrArgMemOnly]>; 1591 1592class SVE_gather_prf_SV 1593 : DefaultAttrsIntrinsic<[], 1594 [ 1595 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1596 llvm_ptr_ty, // Base address 1597 llvm_anyvector_ty, // Offsets 1598 llvm_i32_ty // Prfop 1599 ], 1600 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>; 1601 1602class SVE_gather_prf_VS 1603 : DefaultAttrsIntrinsic<[], 1604 [ 1605 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1606 llvm_anyvector_ty, // Base addresses 1607 llvm_i64_ty, // Scalar offset 1608 llvm_i32_ty // Prfop 1609 ], 1610 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>; 1611 1612class SVE_MatMul_Intrinsic 1613 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1614 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], 1615 [IntrNoMem]>; 1616 1617class SVE_4Vec_BF16 1618 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 1619 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 1620 [IntrNoMem]>; 1621 1622class SVE_4Vec_BF16_Indexed 1623 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 1624 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i32_ty], 1625 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1626 1627// 1628// Loads 1629// 1630 1631def int_aarch64_sve_ld1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1632 1633def int_aarch64_sve_ld2_sret : AdvSIMD_2Vec_PredLoad_Intrinsic; 1634def int_aarch64_sve_ld3_sret : AdvSIMD_3Vec_PredLoad_Intrinsic; 1635def int_aarch64_sve_ld4_sret : AdvSIMD_4Vec_PredLoad_Intrinsic; 1636 1637def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1638def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; 1639def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; 1640 1641def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic; 1642def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic; 1643 1644// 1645// Stores 1646// 1647 1648def int_aarch64_sve_st1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1649def int_aarch64_sve_st2 : AdvSIMD_2Vec_PredStore_Intrinsic; 1650def int_aarch64_sve_st3 : AdvSIMD_3Vec_PredStore_Intrinsic; 1651def int_aarch64_sve_st4 : AdvSIMD_4Vec_PredStore_Intrinsic; 1652 1653def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1654 1655// 1656// Prefetches 1657// 1658 1659def int_aarch64_sve_prf 1660 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty], 1661 [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>; 1662 1663// Scalar + 32-bit scaled offset vector, zero extend, packed and 1664// unpacked. 1665def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV; 1666def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV; 1667def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV; 1668def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV; 1669 1670// Scalar + 32-bit scaled offset vector, sign extend, packed and 1671// unpacked. 1672def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV; 1673def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV; 1674def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV; 1675def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV; 1676 1677// Scalar + 64-bit scaled offset vector. 1678def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV; 1679def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV; 1680def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV; 1681def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV; 1682 1683// Vector + scalar. 1684def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS; 1685def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS; 1686def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS; 1687def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS; 1688 1689// 1690// Scalar to vector operations 1691// 1692 1693def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic; 1694def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic; 1695 1696def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic; 1697 1698// 1699// Address calculation 1700// 1701 1702def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic; 1703def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic; 1704def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic; 1705def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic; 1706 1707// 1708// Integer arithmetic 1709// 1710 1711def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic; 1712def int_aarch64_sve_add_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1713def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; 1714def int_aarch64_sve_sub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1715def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic; 1716 1717def int_aarch64_sve_pmul : AdvSIMD_2VectorArg_Intrinsic; 1718 1719def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic; 1720def int_aarch64_sve_mul_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1721def int_aarch64_sve_mul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1722def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1723def int_aarch64_sve_smulh_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1724def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1725def int_aarch64_sve_umulh_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1726 1727def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1728def int_aarch64_sve_sdiv_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1729def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1730def int_aarch64_sve_udiv_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1731def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1732def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1733 1734def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic; 1735def int_aarch64_sve_smax_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1736def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic; 1737def int_aarch64_sve_umax_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1738def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic; 1739def int_aarch64_sve_smin_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1740def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic; 1741def int_aarch64_sve_umin_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1742def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1743def int_aarch64_sve_sabd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1744def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1745def int_aarch64_sve_uabd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1746 1747def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; 1748def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; 1749def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; 1750def int_aarch64_sve_mla_u : AdvSIMD_Pred3VectorArg_Intrinsic; 1751def int_aarch64_sve_mla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1752def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; 1753def int_aarch64_sve_mls_u : AdvSIMD_Pred3VectorArg_Intrinsic; 1754def int_aarch64_sve_mls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1755 1756def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1757def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1758 1759def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1760def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1761def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic; 1762def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic; 1763 1764def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic; 1765def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic; 1766def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic; 1767 1768def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; 1769def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; 1770 1771def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic; 1772def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1773 1774def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; 1775def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1776 1777def int_aarch64_sve_sqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1778def int_aarch64_sve_sqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1779def int_aarch64_sve_uqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1780def int_aarch64_sve_uqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1781 1782def int_aarch64_sve_orqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1783def int_aarch64_sve_eorqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1784def int_aarch64_sve_andqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1785def int_aarch64_sve_addqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1786def int_aarch64_sve_smaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1787def int_aarch64_sve_umaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1788def int_aarch64_sve_sminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1789def int_aarch64_sve_uminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1790 1791 1792// Shifts 1793 1794def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic; 1795def int_aarch64_sve_asr_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1796def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1797def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic; 1798def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic; 1799def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic; 1800def int_aarch64_sve_lsl_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1801def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1802def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic; 1803def int_aarch64_sve_lsr_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1804def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1805 1806// 1807// Integer comparisons 1808// 1809 1810def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1811def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic; 1812def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1813def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic; 1814def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic; 1815def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic; 1816 1817def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1818def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1819def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1820def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1821def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1822def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1823def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1824def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1825def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1826def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1827 1828// 1829// Counting bits 1830// 1831 1832def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic; 1833def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic; 1834def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; 1835 1836// 1837// Counting elements 1838// 1839 1840def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic; 1841def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic; 1842def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic; 1843def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic; 1844 1845def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; 1846 1847// 1848// FFR manipulation 1849// 1850 1851def int_aarch64_sve_rdffr : ClangBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; 1852def int_aarch64_sve_rdffr_z : ClangBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>; 1853def int_aarch64_sve_setffr : ClangBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>; 1854def int_aarch64_sve_wrffr : ClangBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>; 1855 1856// 1857// Saturating scalar arithmetic 1858// 1859 1860def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1861def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1862def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1863def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1864 1865def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1866def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1867def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1868def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1869def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1870def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1871def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1872def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1873def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1874def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1875 1876def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1877def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1878def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1879def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1880 1881def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1882def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1883def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1884def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1885def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1886def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1887def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1888def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1889def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1890def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1891 1892def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1893def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1894def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1895def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1896 1897def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1898def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1899def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1900def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1901def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1902def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1903def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1904def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1905def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1906def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1907 1908def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1909def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1910def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1911def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1912 1913def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1914def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1915def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1916def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1917def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1918def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1919def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1920def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1921def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1922def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1923 1924// 1925// Reversal 1926// 1927 1928def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic; 1929def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic; 1930def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic; 1931def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic; 1932 1933// 1934// Permutations and selection 1935// 1936 1937def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic; 1938def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1939def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic; 1940def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1941def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic; 1942def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic; 1943def int_aarch64_sve_dup_laneq : SVE2_1VectorArgIndexed_Intrinsic; 1944def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic; 1945def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic; 1946def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; 1947def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; 1948def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; 1949def int_aarch64_sve_rev_b16 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; 1950def int_aarch64_sve_rev_b32 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; 1951def int_aarch64_sve_rev_b64 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; 1952def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; 1953def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1954def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1955def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; 1956def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; 1957def int_aarch64_sve_trn1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1958def int_aarch64_sve_trn1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1959def int_aarch64_sve_trn1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1960def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; 1961def int_aarch64_sve_trn2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1962def int_aarch64_sve_trn2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1963def int_aarch64_sve_trn2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1964def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic; 1965def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic; 1966def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1967def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1968def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; 1969def int_aarch64_sve_uzp1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1970def int_aarch64_sve_uzp1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1971def int_aarch64_sve_uzp1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1972def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; 1973def int_aarch64_sve_uzp2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1974def int_aarch64_sve_uzp2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1975def int_aarch64_sve_uzp2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1976def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic; 1977def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic; 1978def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; 1979def int_aarch64_sve_zip1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1980def int_aarch64_sve_zip1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1981def int_aarch64_sve_zip1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1982def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; 1983def int_aarch64_sve_zip2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1984def int_aarch64_sve_zip2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1985def int_aarch64_sve_zip2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1986def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic; 1987def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic; 1988 1989// 1990// Logical operations 1991// 1992 1993def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic; 1994def int_aarch64_sve_and_u: AdvSIMD_Pred2VectorArg_Intrinsic; 1995def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic; 1996def int_aarch64_sve_bic_u: AdvSIMD_Pred2VectorArg_Intrinsic; 1997def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic; 1998def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic; 1999def int_aarch64_sve_eor_u: AdvSIMD_Pred2VectorArg_Intrinsic; 2000def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic; 2001def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic; 2002def int_aarch64_sve_orr_u: AdvSIMD_Pred2VectorArg_Intrinsic; 2003 2004// 2005// Conversion 2006// 2007 2008def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 2009def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic; 2010def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 2011def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 2012def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic; 2013def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 2014 2015// 2016// While comparisons 2017// 2018 2019def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic; 2020def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic; 2021def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic; 2022def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic; 2023def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic; 2024def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic; 2025def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic; 2026def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic; 2027 2028// 2029// Floating-point arithmetic 2030// 2031 2032def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic; 2033def int_aarch64_sve_fabd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2034def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2035def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2036def int_aarch64_sve_fadd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2037def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic; 2038def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic; 2039def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2040def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 2041def int_aarch64_sve_fdiv_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2042def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 2043def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic; 2044def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic; 2045def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic; 2046def int_aarch64_sve_fmax_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2047def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic; 2048def int_aarch64_sve_fmaxnm_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2049def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic; 2050def int_aarch64_sve_fmin_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2051def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic; 2052def int_aarch64_sve_fminnm_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2053def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic; 2054def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2055def int_aarch64_sve_fmla_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2056def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic; 2057def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2058def int_aarch64_sve_fmls_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2059def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 2060def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic; 2061def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2062def int_aarch64_sve_fmul_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2063def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic; 2064def int_aarch64_sve_fmulx_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2065def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2066def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic; 2067def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic; 2068def int_aarch64_sve_fnmla_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2069def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic; 2070def int_aarch64_sve_fnmls_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2071def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 2072def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic; 2073def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic; 2074def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic; 2075def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic; 2076def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic; 2077def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic; 2078def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic; 2079def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic; 2080def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic; 2081def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic; 2082def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic; 2083def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic; 2084def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic; 2085def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic; 2086def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2087def int_aarch64_sve_fsub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2088def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2089def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic; 2090def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic; 2091def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic; 2092 2093// 2094// Floating-point reductions 2095// 2096 2097def int_aarch64_sve_fadda : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 2098def int_aarch64_sve_faddv : AdvSIMD_SVE_Reduce_Intrinsic; 2099def int_aarch64_sve_fmaxv : AdvSIMD_SVE_Reduce_Intrinsic; 2100def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic; 2101def int_aarch64_sve_fminv : AdvSIMD_SVE_Reduce_Intrinsic; 2102def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic; 2103 2104def int_aarch64_sve_faddqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2105def int_aarch64_sve_fmaxnmqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2106def int_aarch64_sve_fminnmqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2107def int_aarch64_sve_fmaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2108def int_aarch64_sve_fminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2109 2110// 2111// Floating-point conversions 2112// 2113 2114def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic; 2115def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic; 2116def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic; 2117def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 2118def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 2119 2120// 2121// Floating-point comparisons 2122// 2123 2124def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic; 2125def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic; 2126 2127def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic; 2128def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic; 2129def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic; 2130def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic; 2131def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic; 2132 2133def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2134def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2135def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2136def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2137 2138def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 2139def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 2140 2141def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2142def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2143def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2144def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2145 2146def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 2147def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2148def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2149 2150def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2151def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2152def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2153 2154def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2155def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2156def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 2157def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2158 2159def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2160def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2161 2162def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 2163def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2164def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2165def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 2166 2167def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 2168def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2169def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2170def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 2171 2172// 2173// Predicate creation 2174// 2175 2176def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic; 2177 2178// 2179// Predicate operations 2180// 2181 2182def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2183def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2184def int_aarch64_sve_brka : AdvSIMD_Merged1VectorArg_Intrinsic; 2185def int_aarch64_sve_brka_z : AdvSIMD_Pred1VectorArg_Intrinsic; 2186def int_aarch64_sve_brkb : AdvSIMD_Merged1VectorArg_Intrinsic; 2187def int_aarch64_sve_brkb_z : AdvSIMD_Pred1VectorArg_Intrinsic; 2188def int_aarch64_sve_brkn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2189def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2190def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2191def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2192def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2193def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2194def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2195def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2196def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic; 2197def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic; 2198def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; 2199def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; 2200 2201// 2202// Testing predicates 2203// 2204 2205def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic; 2206def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic; 2207def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic; 2208 2209// 2210// Reinterpreting data 2211// 2212 2213def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_any_ty], 2214 [llvm_nxv16i1_ty], 2215 [IntrNoMem]>; 2216 2217def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 2218 [llvm_any_ty], 2219 [IntrNoMem]>; 2220 2221// 2222// Gather loads: scalar base + vector offsets 2223// 2224 2225// 64 bit unscaled offsets 2226def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2227 2228// 64 bit scaled offsets 2229def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2230 2231// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2232def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2233def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2234 2235// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2236def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2237def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2238 2239// 128-bit loads, scaled offsets (indices) 2240def int_aarch64_sve_ld1q_gather_index : AdvSIMD_GatherLoadQ_SV_Intrinsic; 2241 2242// 128-bit loads, unscaled offsets 2243def int_aarch64_sve_ld1q_gather_vector_offset : AdvSIMD_GatherLoadQ_SV_Intrinsic; 2244 2245// 2246// Gather loads: vector base + scalar offset 2247// 2248 2249def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 2250 2251// 128-bit loads, unscaled offsets 2252def int_aarch64_sve_ld1q_gather_scalar_offset : AdvSIMD_GatherLoadQ_VS_Intrinsic; 2253 2254// 2255// First-faulting gather loads: scalar base + vector offsets 2256// 2257 2258// 64 bit unscaled offsets 2259def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; 2260 2261// 64 bit scaled offsets 2262def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; 2263 2264// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 2265def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2266def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2267 2268// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 2269def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2270def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2271 2272// 2273// First-faulting gather loads: vector base + scalar offset 2274// 2275 2276def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic; 2277 2278 2279// 2280// Non-temporal gather loads: scalar base + vector offsets 2281// 2282 2283// 64 bit unscaled offsets 2284def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2285 2286// 64 bit indices 2287def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2288 2289// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2290def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2291 2292// 2293// Non-temporal gather loads: vector base + scalar offset 2294// 2295 2296def int_aarch64_sve_ldnt1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 2297 2298// 2299// Scatter stores: scalar base + vector offsets 2300// 2301 2302// 64 bit unscaled offsets 2303def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2304 2305// 64 bit scaled offsets 2306def int_aarch64_sve_st1_scatter_index 2307 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2308 2309// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2310def int_aarch64_sve_st1_scatter_sxtw 2311 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2312 2313def int_aarch64_sve_st1_scatter_uxtw 2314 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2315 2316// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2317def int_aarch64_sve_st1_scatter_sxtw_index 2318 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2319 2320def int_aarch64_sve_st1_scatter_uxtw_index 2321 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2322 2323// 128-bit stores, scaled offsets (indices) 2324def int_aarch64_sve_st1q_scatter_index : AdvSIMD_ScatterStoreQ_SV_Intrinsic; 2325 2326// 128-bit stores, unscaled offsets 2327def int_aarch64_sve_st1q_scatter_vector_offset : AdvSIMD_ScatterStoreQ_SV_Intrinsic; 2328 2329// 2330// Scatter stores: vector base + scalar offset 2331// 2332 2333def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2334 2335// 128-bit stores, unscaled offsets 2336def int_aarch64_sve_st1q_scatter_scalar_offset : AdvSIMD_ScatterStoreQ_VS_Intrinsic; 2337 2338// 2339// Non-temporal scatter stores: scalar base + vector offsets 2340// 2341 2342// 64 bit unscaled offsets 2343def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2344 2345// 64 bit indices 2346def int_aarch64_sve_stnt1_scatter_index 2347 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2348 2349// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2350def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2351 2352// 2353// Non-temporal scatter stores: vector base + scalar offset 2354// 2355 2356def int_aarch64_sve_stnt1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2357 2358// 2359// SVE2 - Uniform DSP operations 2360// 2361 2362def int_aarch64_sve_saba : AdvSIMD_3VectorArg_Intrinsic; 2363def int_aarch64_sve_shadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2364def int_aarch64_sve_shsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2365def int_aarch64_sve_shsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2366def int_aarch64_sve_sli : AdvSIMD_2VectorArgIndexed_Intrinsic; 2367def int_aarch64_sve_sqabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2368def int_aarch64_sve_sqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2369def int_aarch64_sve_sqdmulh : AdvSIMD_2VectorArg_Intrinsic; 2370def int_aarch64_sve_sqdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2371def int_aarch64_sve_sqneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2372def int_aarch64_sve_sqrdmlah : AdvSIMD_3VectorArg_Intrinsic; 2373def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2374def int_aarch64_sve_sqrdmlsh : AdvSIMD_3VectorArg_Intrinsic; 2375def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2376def int_aarch64_sve_sqrdmulh : AdvSIMD_2VectorArg_Intrinsic; 2377def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2378def int_aarch64_sve_sqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2379def int_aarch64_sve_sqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2380def int_aarch64_sve_sqshlu : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2381def int_aarch64_sve_sqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2382def int_aarch64_sve_sqsub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2383def int_aarch64_sve_sqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2384def int_aarch64_sve_srhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2385def int_aarch64_sve_sri : AdvSIMD_2VectorArgIndexed_Intrinsic; 2386def int_aarch64_sve_srshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2387def int_aarch64_sve_srshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2388def int_aarch64_sve_srsra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2389def int_aarch64_sve_ssra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2390def int_aarch64_sve_suqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2391def int_aarch64_sve_uaba : AdvSIMD_3VectorArg_Intrinsic; 2392def int_aarch64_sve_uhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2393def int_aarch64_sve_uhsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2394def int_aarch64_sve_uhsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2395def int_aarch64_sve_uqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2396def int_aarch64_sve_uqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2397def int_aarch64_sve_uqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2398def int_aarch64_sve_uqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2399def int_aarch64_sve_uqsub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2400def int_aarch64_sve_uqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2401def int_aarch64_sve_urecpe : AdvSIMD_Merged1VectorArg_Intrinsic; 2402def int_aarch64_sve_urhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2403def int_aarch64_sve_urshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2404def int_aarch64_sve_urshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2405def int_aarch64_sve_ursqrte : AdvSIMD_Merged1VectorArg_Intrinsic; 2406def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2407def int_aarch64_sve_usqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2408def int_aarch64_sve_usra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2409 2410// 2411// SVE2 - Widening DSP operations 2412// 2413 2414def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic; 2415def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic; 2416def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic; 2417def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic; 2418def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic; 2419def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic; 2420def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic; 2421def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic; 2422def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic; 2423def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic; 2424def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic; 2425def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic; 2426def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic; 2427def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic; 2428def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic; 2429def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic; 2430def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic; 2431def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic; 2432def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic; 2433def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic; 2434def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic; 2435def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic; 2436def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic; 2437def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic; 2438def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic; 2439def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic; 2440def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic; 2441def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic; 2442 2443// 2444// SVE2 - Non-widening pairwise arithmetic 2445// 2446 2447def int_aarch64_sve_addp : AdvSIMD_Pred2VectorArg_Intrinsic; 2448def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic; 2449def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2450def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2451def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2452def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2453def int_aarch64_sve_smaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2454def int_aarch64_sve_sminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2455def int_aarch64_sve_umaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2456def int_aarch64_sve_uminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2457 2458// 2459// SVE2 - Widening pairwise arithmetic 2460// 2461 2462def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2463def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2464 2465// 2466// SVE2 - Uniform complex integer arithmetic 2467// 2468 2469def int_aarch64_sve_cadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2470def int_aarch64_sve_sqcadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2471def int_aarch64_sve_cmla_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2472def int_aarch64_sve_cmla_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2473def int_aarch64_sve_sqrdcmlah_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2474def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2475 2476// 2477// SVE2 - Widening complex integer arithmetic 2478// 2479 2480def int_aarch64_sve_saddlbt : SVE2_2VectorArg_Long_Intrinsic; 2481def int_aarch64_sve_ssublbt : SVE2_2VectorArg_Long_Intrinsic; 2482def int_aarch64_sve_ssubltb : SVE2_2VectorArg_Long_Intrinsic; 2483 2484// 2485// SVE2 - Widening complex integer dot product 2486// 2487 2488def int_aarch64_sve_cdot : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2489def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic; 2490 2491// 2492// SVE2 - Floating-point widening multiply-accumulate 2493// 2494 2495def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic; 2496def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2497def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic; 2498def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2499def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic; 2500def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2501def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic; 2502def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2503 2504// 2505// SVE2 - Floating-point integer binary logarithm 2506// 2507 2508def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic; 2509 2510// 2511// SVE2 - Vector histogram count 2512// 2513 2514def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic; 2515def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic; 2516 2517// 2518// SVE2 - Character match 2519// 2520 2521def int_aarch64_sve_match : AdvSIMD_SVE_Compare_Intrinsic; 2522def int_aarch64_sve_nmatch : AdvSIMD_SVE_Compare_Intrinsic; 2523 2524// 2525// SVE2 - Unary narrowing operations 2526// 2527 2528def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2529def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2530def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic; 2531def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2532def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2533def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2534 2535// 2536// SVE2 - Binary narrowing DSP operations 2537// 2538def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2539def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2540 2541def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2542def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2543 2544def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2545def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2546 2547def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2548def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2549 2550// Narrowing shift right 2551def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2552def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2553 2554def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2555def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2556 2557// Saturating shift right - signed input/output 2558def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2559def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2560 2561def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2562def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2563 2564// Saturating shift right - unsigned input/output 2565def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2566def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2567 2568def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2569def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2570 2571// Saturating shift right - signed input, unsigned output 2572def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2573def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2574 2575def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2576def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2577 2578// SVE2 MLA LANE. 2579def int_aarch64_sve_smlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2580def int_aarch64_sve_smlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2581def int_aarch64_sve_umlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2582def int_aarch64_sve_umlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2583def int_aarch64_sve_smlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2584def int_aarch64_sve_smlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2585def int_aarch64_sve_umlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2586def int_aarch64_sve_umlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2587def int_aarch64_sve_smullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2588def int_aarch64_sve_smullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2589def int_aarch64_sve_umullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2590def int_aarch64_sve_umullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2591def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2592def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2593def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2594def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2595def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2596def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2597 2598// SVE2 MLA Unpredicated. 2599def int_aarch64_sve_smlalb : SVE2_3VectorArg_Long_Intrinsic; 2600def int_aarch64_sve_smlalt : SVE2_3VectorArg_Long_Intrinsic; 2601def int_aarch64_sve_umlalb : SVE2_3VectorArg_Long_Intrinsic; 2602def int_aarch64_sve_umlalt : SVE2_3VectorArg_Long_Intrinsic; 2603def int_aarch64_sve_smlslb : SVE2_3VectorArg_Long_Intrinsic; 2604def int_aarch64_sve_smlslt : SVE2_3VectorArg_Long_Intrinsic; 2605def int_aarch64_sve_umlslb : SVE2_3VectorArg_Long_Intrinsic; 2606def int_aarch64_sve_umlslt : SVE2_3VectorArg_Long_Intrinsic; 2607def int_aarch64_sve_smullb : SVE2_2VectorArg_Long_Intrinsic; 2608def int_aarch64_sve_smullt : SVE2_2VectorArg_Long_Intrinsic; 2609def int_aarch64_sve_umullb : SVE2_2VectorArg_Long_Intrinsic; 2610def int_aarch64_sve_umullt : SVE2_2VectorArg_Long_Intrinsic; 2611 2612def int_aarch64_sve_sqdmlalb : SVE2_3VectorArg_Long_Intrinsic; 2613def int_aarch64_sve_sqdmlalt : SVE2_3VectorArg_Long_Intrinsic; 2614def int_aarch64_sve_sqdmlslb : SVE2_3VectorArg_Long_Intrinsic; 2615def int_aarch64_sve_sqdmlslt : SVE2_3VectorArg_Long_Intrinsic; 2616def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic; 2617def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic; 2618def int_aarch64_sve_sqdmlalbt : SVE2_3VectorArg_Long_Intrinsic; 2619def int_aarch64_sve_sqdmlslbt : SVE2_3VectorArg_Long_Intrinsic; 2620 2621// SVE2 ADDSUB Long Unpredicated. 2622def int_aarch64_sve_adclb : AdvSIMD_3VectorArg_Intrinsic; 2623def int_aarch64_sve_adclt : AdvSIMD_3VectorArg_Intrinsic; 2624def int_aarch64_sve_sbclb : AdvSIMD_3VectorArg_Intrinsic; 2625def int_aarch64_sve_sbclt : AdvSIMD_3VectorArg_Intrinsic; 2626 2627// 2628// SVE2 - Polynomial arithmetic 2629// 2630def int_aarch64_sve_eorbt : AdvSIMD_3VectorArg_Intrinsic; 2631def int_aarch64_sve_eortb : AdvSIMD_3VectorArg_Intrinsic; 2632def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic; 2633def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic; 2634 2635// 2636// SVE2 bitwise ternary operations. 2637// 2638def int_aarch64_sve_eor3 : AdvSIMD_3VectorArg_Intrinsic; 2639def int_aarch64_sve_bcax : AdvSIMD_3VectorArg_Intrinsic; 2640def int_aarch64_sve_bsl : AdvSIMD_3VectorArg_Intrinsic; 2641def int_aarch64_sve_bsl1n : AdvSIMD_3VectorArg_Intrinsic; 2642def int_aarch64_sve_bsl2n : AdvSIMD_3VectorArg_Intrinsic; 2643def int_aarch64_sve_nbsl : AdvSIMD_3VectorArg_Intrinsic; 2644def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic; 2645 2646// 2647// SVE2 - Optional AES, SHA-3 and SM4 2648// 2649 2650def int_aarch64_sve_aesd : ClangBuiltin<"__builtin_sve_svaesd_u8">, 2651 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2652 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2653 [IntrNoMem]>; 2654def int_aarch64_sve_aesimc : ClangBuiltin<"__builtin_sve_svaesimc_u8">, 2655 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2656 [llvm_nxv16i8_ty], 2657 [IntrNoMem]>; 2658def int_aarch64_sve_aese : ClangBuiltin<"__builtin_sve_svaese_u8">, 2659 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2660 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2661 [IntrNoMem]>; 2662def int_aarch64_sve_aesmc : ClangBuiltin<"__builtin_sve_svaesmc_u8">, 2663 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2664 [llvm_nxv16i8_ty], 2665 [IntrNoMem]>; 2666def int_aarch64_sve_rax1 : ClangBuiltin<"__builtin_sve_svrax1_u64">, 2667 DefaultAttrsIntrinsic<[llvm_nxv2i64_ty], 2668 [llvm_nxv2i64_ty, llvm_nxv2i64_ty], 2669 [IntrNoMem]>; 2670def int_aarch64_sve_sm4e : ClangBuiltin<"__builtin_sve_svsm4e_u32">, 2671 DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], 2672 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2673 [IntrNoMem]>; 2674def int_aarch64_sve_sm4ekey : ClangBuiltin<"__builtin_sve_svsm4ekey_u32">, 2675 DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], 2676 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2677 [IntrNoMem]>; 2678// 2679// SVE2 - Extended table lookup/permute 2680// 2681 2682def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic; 2683def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic; 2684 2685// 2686// SVE2 - Optional bit permutation 2687// 2688 2689def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic; 2690def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic; 2691def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic; 2692 2693 2694// 2695// SVE ACLE: 7.3. INT8 matrix multiply extensions 2696// 2697def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic; 2698def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic; 2699def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic; 2700 2701def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic; 2702def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2703def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2704 2705// 2706// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions 2707// 2708def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; 2709 2710// 2711// SVE ACLE: 7.2. BFloat16 extensions 2712// 2713 2714def int_aarch64_sve_bfdot : SVE_4Vec_BF16; 2715def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16; 2716def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16; 2717 2718def int_aarch64_sve_bfmmla : SVE_4Vec_BF16; 2719 2720def int_aarch64_sve_bfdot_lane_v2 : SVE_4Vec_BF16_Indexed; 2721def int_aarch64_sve_bfmlalb_lane_v2 : SVE_4Vec_BF16_Indexed; 2722def int_aarch64_sve_bfmlalt_lane_v2 : SVE_4Vec_BF16_Indexed; 2723 2724// 2725// SVE2.1 - Contiguous loads to multiple consecutive vectors 2726// 2727 2728 class SVE2p1_Load_PN_X2_Intrinsic 2729 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2730 [llvm_aarch64_svcount_ty, llvm_ptr_ty], 2731 [IntrReadMem, IntrArgMemOnly]>; 2732 2733 class SVE2p1_Load_PN_X4_Intrinsic 2734 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 2735 LLVMMatchType<0>, LLVMMatchType<0>], 2736 [llvm_aarch64_svcount_ty, llvm_ptr_ty], 2737 [IntrReadMem, IntrArgMemOnly]>; 2738 2739def int_aarch64_sve_ld1_pn_x2 : SVE2p1_Load_PN_X2_Intrinsic; 2740def int_aarch64_sve_ld1_pn_x4 : SVE2p1_Load_PN_X4_Intrinsic; 2741def int_aarch64_sve_ldnt1_pn_x2 : SVE2p1_Load_PN_X2_Intrinsic; 2742def int_aarch64_sve_ldnt1_pn_x4 : SVE2p1_Load_PN_X4_Intrinsic; 2743 2744// 2745// SVE2.1 - Contiguous loads to quadword (single vector) 2746// 2747 2748class SVE2p1_Single_Load_Quadword 2749 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2750 [llvm_nxv1i1_ty, llvm_ptr_ty], 2751 [IntrReadMem, IntrArgMemOnly]>; 2752def int_aarch64_sve_ld1uwq : SVE2p1_Single_Load_Quadword; 2753def int_aarch64_sve_ld1udq : SVE2p1_Single_Load_Quadword; 2754 2755// 2756// SVE2.1 - Contiguous store from quadword (single vector) 2757// 2758 2759class SVE2p1_Single_Store_Quadword 2760 : DefaultAttrsIntrinsic<[], 2761 [llvm_anyvector_ty, llvm_nxv1i1_ty, llvm_ptr_ty], 2762 [IntrWriteMem, IntrArgMemOnly]>; 2763def int_aarch64_sve_st1wq : SVE2p1_Single_Store_Quadword; 2764def int_aarch64_sve_st1dq : SVE2p1_Single_Store_Quadword; 2765 2766 2767def int_aarch64_sve_ld2q_sret : AdvSIMD_2Vec_PredLoad_Intrinsic; 2768def int_aarch64_sve_ld3q_sret : AdvSIMD_3Vec_PredLoad_Intrinsic; 2769def int_aarch64_sve_ld4q_sret : AdvSIMD_4Vec_PredLoad_Intrinsic; 2770 2771def int_aarch64_sve_st2q : AdvSIMD_2Vec_PredStore_Intrinsic; 2772def int_aarch64_sve_st3q : AdvSIMD_3Vec_PredStore_Intrinsic; 2773def int_aarch64_sve_st4q : AdvSIMD_4Vec_PredStore_Intrinsic; 2774 2775// 2776// SVE2.1 - Contiguous stores to multiple consecutive vectors 2777// 2778 2779 class SVE2p1_Store_PN_X2_Intrinsic 2780 : DefaultAttrsIntrinsic<[], [ llvm_anyvector_ty, LLVMMatchType<0>, 2781 llvm_aarch64_svcount_ty, llvm_ptr_ty ], 2782 [IntrWriteMem, IntrArgMemOnly]>; 2783 2784 class SVE2p1_Store_PN_X4_Intrinsic 2785 : DefaultAttrsIntrinsic<[], [ llvm_anyvector_ty, LLVMMatchType<0>, 2786 LLVMMatchType<0>, LLVMMatchType<0>, 2787 llvm_aarch64_svcount_ty, llvm_ptr_ty], 2788 [IntrWriteMem, IntrArgMemOnly]>; 2789 2790def int_aarch64_sve_st1_pn_x2 : SVE2p1_Store_PN_X2_Intrinsic; 2791def int_aarch64_sve_st1_pn_x4 : SVE2p1_Store_PN_X4_Intrinsic; 2792def int_aarch64_sve_stnt1_pn_x2 : SVE2p1_Store_PN_X2_Intrinsic; 2793def int_aarch64_sve_stnt1_pn_x4 : SVE2p1_Store_PN_X4_Intrinsic; 2794} 2795 2796// 2797// SVE2 - Contiguous conflict detection 2798// 2799 2800def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic; 2801def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic; 2802def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic; 2803def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic; 2804def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic; 2805def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic; 2806def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic; 2807def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic; 2808 2809// Scalable Matrix Extension (SME) Intrinsics 2810let TargetPrefix = "aarch64" in { 2811 class SME_Load_Store_Intrinsic<LLVMType pred_ty> 2812 : DefaultAttrsIntrinsic<[], 2813 [pred_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; 2814 2815 // Loads 2816 def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2817 def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2818 def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2819 def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2820 def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2821 def int_aarch64_sme_ld1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2822 def int_aarch64_sme_ld1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2823 def int_aarch64_sme_ld1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2824 def int_aarch64_sme_ld1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2825 def int_aarch64_sme_ld1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2826 2827 // Stores 2828 def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2829 def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2830 def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2831 def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2832 def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2833 def int_aarch64_sme_st1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2834 def int_aarch64_sme_st1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2835 def int_aarch64_sme_st1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2836 def int_aarch64_sme_st1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2837 def int_aarch64_sme_st1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2838 2839 // Spill + fill 2840 class SME_LDR_STR_ZA_Intrinsic 2841 : DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty]>; 2842 def int_aarch64_sme_ldr : SME_LDR_STR_ZA_Intrinsic; 2843 def int_aarch64_sme_str : SME_LDR_STR_ZA_Intrinsic; 2844 2845 class SME_TileToVector_Intrinsic 2846 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2847 [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2848 llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; 2849 class SME_VectorToTile_Intrinsic 2850 : DefaultAttrsIntrinsic<[], 2851 [llvm_i32_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2852 llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>; 2853 2854 def int_aarch64_sme_read_horiz : SME_TileToVector_Intrinsic; 2855 def int_aarch64_sme_read_vert : SME_TileToVector_Intrinsic; 2856 def int_aarch64_sme_write_horiz : SME_VectorToTile_Intrinsic; 2857 def int_aarch64_sme_write_vert : SME_VectorToTile_Intrinsic; 2858 2859 def int_aarch64_sme_readq_horiz : SME_TileToVector_Intrinsic; 2860 def int_aarch64_sme_readq_vert : SME_TileToVector_Intrinsic; 2861 def int_aarch64_sme_writeq_horiz : SME_VectorToTile_Intrinsic; 2862 def int_aarch64_sme_writeq_vert : SME_VectorToTile_Intrinsic; 2863 2864 class SME_MOVAZ_TileToVector_X2_Intrinsic 2865 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2866 [llvm_i32_ty, llvm_i32_ty], 2867 [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>; 2868 2869 class SME_MOVAZ_TileToVector_X4_Intrinsic 2870 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 2871 LLVMMatchType<0>,LLVMMatchType<0>], 2872 [llvm_i32_ty, llvm_i32_ty], 2873 [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>; 2874 2875 def int_aarch64_sme_readz_horiz_x2 : SME_MOVAZ_TileToVector_X2_Intrinsic; 2876 def int_aarch64_sme_readz_vert_x2 : SME_MOVAZ_TileToVector_X2_Intrinsic; 2877 2878 def int_aarch64_sme_readz_horiz_x4 : SME_MOVAZ_TileToVector_X4_Intrinsic; 2879 def int_aarch64_sme_readz_vert_x4 : SME_MOVAZ_TileToVector_X4_Intrinsic; 2880 2881 class SME_MOVAZ_TileToVector_Intrinsic 2882 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2883 [llvm_i32_ty, llvm_i32_ty], 2884 [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>; 2885 2886 def int_aarch64_sme_readz_horiz : SME_MOVAZ_TileToVector_Intrinsic; 2887 def int_aarch64_sme_readz_vert : SME_MOVAZ_TileToVector_Intrinsic; 2888 2889 def int_aarch64_sme_readz_q_horiz : SME_MOVAZ_TileToVector_Intrinsic; 2890 def int_aarch64_sme_readz_q_vert : SME_MOVAZ_TileToVector_Intrinsic; 2891 2892 def int_aarch64_sme_readz_x2 2893 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2894 [llvm_i32_ty], 2895 [IntrNoMem, IntrHasSideEffects]>; 2896 2897 def int_aarch64_sme_readz_x4 2898 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 2899 [llvm_i32_ty], 2900 [IntrNoMem, IntrHasSideEffects]>; 2901 2902 def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; 2903 2904 class SME_OuterProduct_Intrinsic 2905 : DefaultAttrsIntrinsic<[], 2906 [llvm_i32_ty, 2907 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2908 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2909 LLVMMatchType<0>, 2910 llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>; 2911 2912 def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic; 2913 def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic; 2914 2915 def int_aarch64_sme_mopa_wide : SME_OuterProduct_Intrinsic; 2916 def int_aarch64_sme_mops_wide : SME_OuterProduct_Intrinsic; 2917 2918 def int_aarch64_sme_smopa_wide : SME_OuterProduct_Intrinsic; 2919 def int_aarch64_sme_smops_wide : SME_OuterProduct_Intrinsic; 2920 def int_aarch64_sme_umopa_wide : SME_OuterProduct_Intrinsic; 2921 def int_aarch64_sme_umops_wide : SME_OuterProduct_Intrinsic; 2922 def int_aarch64_sme_sumopa_wide : SME_OuterProduct_Intrinsic; 2923 def int_aarch64_sme_sumops_wide : SME_OuterProduct_Intrinsic; 2924 def int_aarch64_sme_usmopa_wide : SME_OuterProduct_Intrinsic; 2925 def int_aarch64_sme_usmops_wide : SME_OuterProduct_Intrinsic; 2926 2927 class SME_AddVectorToTile_Intrinsic 2928 : DefaultAttrsIntrinsic<[], 2929 [llvm_i32_ty, 2930 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2931 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2932 llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>; 2933 2934 def int_aarch64_sme_addha : SME_AddVectorToTile_Intrinsic; 2935 def int_aarch64_sme_addva : SME_AddVectorToTile_Intrinsic; 2936 2937 // 2938 // Counting elements 2939 // 2940 2941 class AdvSIMD_SME_CNTSB_Intrinsic 2942 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem]>; 2943 2944 def int_aarch64_sme_cntsb : AdvSIMD_SME_CNTSB_Intrinsic; 2945 def int_aarch64_sme_cntsh : AdvSIMD_SME_CNTSB_Intrinsic; 2946 def int_aarch64_sme_cntsw : AdvSIMD_SME_CNTSB_Intrinsic; 2947 def int_aarch64_sme_cntsd : AdvSIMD_SME_CNTSB_Intrinsic; 2948 2949 // 2950 // PSTATE Functions 2951 // 2952 2953 def int_aarch64_sme_get_tpidr2 2954 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], 2955 [IntrNoMem, IntrHasSideEffects]>; 2956 def int_aarch64_sme_set_tpidr2 2957 : DefaultAttrsIntrinsic<[], [llvm_i64_ty], 2958 [IntrNoMem, IntrHasSideEffects]>; 2959 2960 def int_aarch64_sme_za_enable 2961 : DefaultAttrsIntrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; 2962 def int_aarch64_sme_za_disable 2963 : DefaultAttrsIntrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; 2964 2965 // Clamp 2966 // 2967 2968 def int_aarch64_sve_sclamp : AdvSIMD_3VectorArg_Intrinsic; 2969 def int_aarch64_sve_uclamp : AdvSIMD_3VectorArg_Intrinsic; 2970 def int_aarch64_sve_fclamp : AdvSIMD_3VectorArg_Intrinsic; 2971 2972 2973 // 2974 // Reversal 2975 // 2976 2977 def int_aarch64_sve_revd : AdvSIMD_Merged1VectorArg_Intrinsic; 2978 2979 // 2980 // Predicate selection 2981 // 2982 2983 def int_aarch64_sve_psel 2984 : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 2985 [llvm_nxv16i1_ty, 2986 llvm_anyvector_ty, llvm_i32_ty], 2987 [IntrNoMem]>; 2988 2989 // 2990 // Predicate-pair intrinsics 2991 // 2992 foreach cmp = ["ge", "gt", "hi", "hs", "le", "lo", "ls", "lt"] in { 2993 def int_aarch64_sve_while # cmp # _x2 2994 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2995 [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 2996 } 2997 2998 // 2999 // Predicate-as-counter intrinsics 3000 // 3001 3002 def int_aarch64_sve_pext 3003 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 3004 [llvm_aarch64_svcount_ty, llvm_i32_ty], 3005 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 3006 3007 def int_aarch64_sve_pext_x2 3008 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3009 [llvm_aarch64_svcount_ty, llvm_i32_ty], 3010 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 3011 3012 def int_aarch64_sve_ptrue_c8 3013 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 3014 def int_aarch64_sve_ptrue_c16 3015 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 3016 def int_aarch64_sve_ptrue_c32 3017 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 3018 def int_aarch64_sve_ptrue_c64 3019 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 3020 3021 def int_aarch64_sve_cntp_c8 3022 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 3023 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 3024 def int_aarch64_sve_cntp_c16 3025 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 3026 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 3027 def int_aarch64_sve_cntp_c32 3028 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 3029 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 3030 def int_aarch64_sve_cntp_c64 3031 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 3032 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 3033 3034 // While (predicate-as-counter) intrinsics 3035 foreach cmp = ["ge", "gt", "hi", "hs", "le", "lo", "ls", "lt"] in { 3036 foreach ty = ["c8", "c16", "c32", "c64"] in { 3037 def int_aarch64_sve_while # cmp # _ # ty 3038 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], 3039 [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], 3040 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 3041 } 3042 } 3043 3044 // 3045 // SME2 Intrinsics 3046 // 3047 3048 class SME2_Matrix_ArrayVector_Single_Single_Intrinsic 3049 : DefaultAttrsIntrinsic<[], 3050 [llvm_i32_ty, 3051 llvm_anyvector_ty, LLVMMatchType<0>], 3052 []>; 3053 3054 class SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic 3055 : DefaultAttrsIntrinsic<[], 3056 [llvm_i32_ty, 3057 llvm_anyvector_ty, LLVMMatchType<0>, 3058 LLVMMatchType<0>], 3059 []>; 3060 3061 class SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic 3062 : DefaultAttrsIntrinsic<[], 3063 [llvm_i32_ty, 3064 llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, 3065 LLVMMatchType<0>], 3066 []>; 3067 3068 class SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic 3069 : DefaultAttrsIntrinsic<[], 3070 [llvm_i32_ty, 3071 llvm_anyvector_ty, LLVMMatchType<0>, 3072 LLVMMatchType<0>, LLVMMatchType<0>], 3073 []>; 3074 3075 class SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic 3076 : DefaultAttrsIntrinsic<[], 3077 [llvm_i32_ty, 3078 llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, 3079 LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3080 []>; 3081 3082 class SME2_Matrix_ArrayVector_Single_Index_Intrinsic 3083 : DefaultAttrsIntrinsic<[], 3084 [llvm_i32_ty, 3085 llvm_anyvector_ty, 3086 LLVMMatchType<0>, llvm_i32_ty], 3087 [ImmArg<ArgIndex<3>>]>; 3088 3089 class SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic 3090 : DefaultAttrsIntrinsic<[], 3091 [llvm_i32_ty, 3092 llvm_anyvector_ty, LLVMMatchType<0>, 3093 LLVMMatchType<0>, llvm_i32_ty], 3094 [ImmArg<ArgIndex<4>>]>; 3095 3096 class SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic 3097 : DefaultAttrsIntrinsic<[], 3098 [llvm_i32_ty, 3099 llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, 3100 LLVMMatchType<0>, llvm_i32_ty], 3101 [ImmArg<ArgIndex<6>>]>; 3102 3103 class SME2_VG2_Multi_Imm_Intrinsic 3104 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 3105 [llvm_anyvector_ty, LLVMMatchType<0>, 3106 llvm_i32_ty], 3107 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 3108 3109 class SME2_VG4_Multi_Imm_Intrinsic 3110 : DefaultAttrsIntrinsic<[LLVMSubdivide4VectorType<0>], 3111 [llvm_anyvector_ty, LLVMMatchType<0>, 3112 LLVMMatchType<0>, LLVMMatchType<0>, 3113 llvm_i32_ty], 3114 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 3115 3116 class SME2_ZA_Write_VG2_Intrinsic 3117 : DefaultAttrsIntrinsic<[], 3118 [llvm_i32_ty, 3119 llvm_anyvector_ty, LLVMMatchType<0>], 3120 []>; 3121 3122 class SME2_ZA_Write_VG4_Intrinsic 3123 : DefaultAttrsIntrinsic<[], 3124 [llvm_i32_ty, 3125 llvm_anyvector_ty, LLVMMatchType<0>, 3126 LLVMMatchType<0>, LLVMMatchType<0>], 3127 []>; 3128 3129 class SME2_VG2_Multi_Single_Intrinsic 3130 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3131 [LLVMMatchType<0>, LLVMMatchType<0>, 3132 LLVMMatchType<0>], 3133 [IntrNoMem]>; 3134 3135 class SME2_VG4_Multi_Single_Intrinsic 3136 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3137 LLVMMatchType<0>, LLVMMatchType<0>], 3138 [LLVMMatchType<0>, LLVMMatchType<0>, 3139 LLVMMatchType<0>, LLVMMatchType<0>, 3140 LLVMMatchType<0>], 3141 [IntrNoMem]>; 3142 3143 class SME2_VG2_Multi_Multi_Intrinsic 3144 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3145 [LLVMMatchType<0>, LLVMMatchType<0>, 3146 LLVMMatchType<0>, LLVMMatchType<0>], 3147 [IntrNoMem]>; 3148 3149 class SME2_VG4_Multi_Multi_Intrinsic 3150 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3151 LLVMMatchType<0>, LLVMMatchType<0>], 3152 [LLVMMatchType<0>, LLVMMatchType<0>, 3153 LLVMMatchType<0>, LLVMMatchType<0>, 3154 LLVMMatchType<0>, LLVMMatchType<0>, 3155 LLVMMatchType<0>, LLVMMatchType<0>], 3156 [IntrNoMem]>; 3157 3158 class SVE2_VG2_Sel_Intrinsic 3159 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3160 [llvm_aarch64_svcount_ty, LLVMMatchType<0>, 3161 LLVMMatchType<0>, LLVMMatchType<0>, 3162 LLVMMatchType<0>], [IntrNoMem]>; 3163 3164 class SVE2_VG4_Sel_Intrinsic 3165 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3166 LLVMMatchType<0>, LLVMMatchType<0>], 3167 [llvm_aarch64_svcount_ty, LLVMMatchType<0>, 3168 LLVMMatchType<0>, LLVMMatchType<0>, 3169 LLVMMatchType<0>, LLVMMatchType<0>, 3170 LLVMMatchType<0>, LLVMMatchType<0>, 3171 LLVMMatchType<0>], [IntrNoMem]>; 3172 3173 class SME2_CVT_VG2_SINGLE_Intrinsic 3174 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 3175 [llvm_anyvector_ty, LLVMMatchType<0>], 3176 [IntrNoMem]>; 3177 3178 class SME2_CVT_VG2_SINGLE_BF16_Intrinsic 3179 : DefaultAttrsIntrinsic<[llvm_nxv8bf16_ty], 3180 [llvm_nxv4f32_ty, llvm_nxv4f32_ty], 3181 [IntrNoMem]>; 3182 3183 class SME2_CVT_WIDENING_VG2_Intrinsic 3184 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3185 [LLVMSubdivide2VectorType<0>], [IntrNoMem]>; 3186 3187 3188 class SME2_CVT_VG4_SINGLE_Intrinsic 3189 : DefaultAttrsIntrinsic<[LLVMSubdivide4VectorType<0>], 3190 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3191 [IntrNoMem]>; 3192 3193 class SME2_CVT_X2_Intrinsic 3194 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3195 [llvm_anyvector_ty, LLVMMatchType<1>], 3196 [IntrNoMem]>; 3197 3198 class SME2_CVT_X4_Intrinsic 3199 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3200 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>], 3201 [IntrNoMem]>; 3202 3203 class SME2_BFMLS_Intrinsic 3204 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 3205 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 3206 [IntrNoMem]>; 3207 3208 class SME2_BFMLS_Lane_Intrinsic 3209 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 3210 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i32_ty], 3211 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 3212 3213 class SME2_ZA_ArrayVector_Read_VG2_Intrinsic 3214 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3215 [llvm_i32_ty], 3216 []>; 3217 3218 class SME2_ZA_ArrayVector_Read_VG4_Intrinsic 3219 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3220 LLVMMatchType<0>, LLVMMatchType<0>], 3221 [llvm_i32_ty], 3222 []>; 3223 3224 class SME2_Matrix_TileVector_Read_VG2_Intrinsic 3225 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3226 [llvm_i32_ty, llvm_i32_ty], 3227 []>; 3228 3229 class SME2_Matrix_TileVector_Read_VG4_Intrinsic 3230 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3231 LLVMMatchType<0>, LLVMMatchType<0>], 3232 [llvm_i32_ty, llvm_i32_ty], 3233 []>; 3234 3235 class SME2_ZA_ArrayVector_Write_VG2_Intrinsic 3236 : DefaultAttrsIntrinsic<[], 3237 [llvm_i32_ty, 3238 llvm_anyvector_ty, LLVMMatchType<0>], 3239 []>; 3240 3241 class SME2_ZA_ArrayVector_Write_VG4_Intrinsic 3242 : DefaultAttrsIntrinsic<[], 3243 [llvm_i32_ty, 3244 llvm_anyvector_ty, LLVMMatchType<0>, 3245 LLVMMatchType<0>, LLVMMatchType<0>], 3246 []>; 3247 3248 class SME2_Matrix_TileVector_Write_VG2_Intrinsic 3249 : DefaultAttrsIntrinsic<[], 3250 [llvm_i32_ty, llvm_i32_ty, 3251 llvm_anyvector_ty, LLVMMatchType<0>], 3252 [ImmArg<ArgIndex<0>>]>; 3253 3254 class SME2_Matrix_TileVector_Write_VG4_Intrinsic 3255 : DefaultAttrsIntrinsic<[], 3256 [llvm_i32_ty, llvm_i32_ty, 3257 llvm_anyvector_ty, LLVMMatchType<0>, 3258 LLVMMatchType<0>, LLVMMatchType<0>], 3259 [ImmArg<ArgIndex<0>>]>; 3260 3261 class SME2_VG2_Multi_Single_Single_Intrinsic 3262 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3263 [LLVMMatchType<0>, LLVMMatchType<0>, 3264 LLVMMatchType<0>, LLVMMatchType<0>], 3265 [IntrNoMem]>; 3266 3267 class SME2_VG4_Multi_Single_Single_Intrinsic 3268 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3269 LLVMMatchType<0>, LLVMMatchType<0>], 3270 [LLVMMatchType<0>, LLVMMatchType<0>, 3271 LLVMMatchType<0>, LLVMMatchType<0>, 3272 LLVMMatchType<0>, LLVMMatchType<0>], 3273 [IntrNoMem]>; 3274 3275 class SVE2_VG2_ZipUzp_Intrinsic 3276 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3277 [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; 3278 3279 class SVE2_VG4_ZipUzp_Intrinsic 3280 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3281 LLVMMatchType<0>, LLVMMatchType<0>], 3282 [LLVMMatchType<0>, LLVMMatchType<0>, 3283 LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; 3284 3285 class SME2_VG2_Unpk_Intrinsic 3286 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3287 [LLVMSubdivide2VectorType<0>], [IntrNoMem]>; 3288 3289 class SME2_VG4_Unpk_Intrinsic 3290 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3291 LLVMMatchType<0>, LLVMMatchType<0>], 3292 [LLVMSubdivide2VectorType<0>, LLVMSubdivide2VectorType<0>], 3293 [IntrNoMem]>; 3294 3295 // 3296 // Multi-vector fused multiply-add/subtract 3297 // 3298 3299 def int_aarch64_sme_fmla_single_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3300 def int_aarch64_sme_fmls_single_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3301 def int_aarch64_sme_fmla_single_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3302 def int_aarch64_sme_fmls_single_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3303 3304 def int_aarch64_sme_fmla_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3305 def int_aarch64_sme_fmls_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3306 def int_aarch64_sme_fmla_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3307 def int_aarch64_sme_fmls_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3308 3309 def int_aarch64_sme_fmla_lane_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3310 def int_aarch64_sme_fmls_lane_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3311 def int_aarch64_sme_fmla_lane_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3312 def int_aarch64_sme_fmls_lane_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3313 3314 // 3315 // Outer product and accumulate/subtract intrinsics 3316 // 3317 3318 def int_aarch64_sme_smopa_za32 : SME_OuterProduct_Intrinsic; 3319 def int_aarch64_sme_umopa_za32 : SME_OuterProduct_Intrinsic; 3320 def int_aarch64_sme_smops_za32 : SME_OuterProduct_Intrinsic; 3321 def int_aarch64_sme_umops_za32 : SME_OuterProduct_Intrinsic; 3322 3323 def int_aarch64_sme_bmopa_za32 : SME_OuterProduct_Intrinsic; 3324 def int_aarch64_sme_bmops_za32 : SME_OuterProduct_Intrinsic; 3325 3326 // 3327 // Multi-vector rounding shift left intrinsics 3328 // 3329 3330 def int_aarch64_sve_srshl_single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3331 def int_aarch64_sve_urshl_single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3332 def int_aarch64_sve_srshl_single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3333 def int_aarch64_sve_urshl_single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3334 3335 def int_aarch64_sve_srshl_x2 : SME2_VG2_Multi_Multi_Intrinsic; 3336 def int_aarch64_sve_urshl_x2 : SME2_VG2_Multi_Multi_Intrinsic; 3337 def int_aarch64_sve_srshl_x4 : SME2_VG4_Multi_Multi_Intrinsic; 3338 def int_aarch64_sve_urshl_x4 : SME2_VG4_Multi_Multi_Intrinsic; 3339 3340 // Multi-vector saturating rounding shift right intrinsics 3341 3342 def int_aarch64_sve_sqrshr_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3343 def int_aarch64_sve_uqrshr_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3344 def int_aarch64_sve_sqrshr_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3345 def int_aarch64_sve_uqrshr_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3346 3347 def int_aarch64_sve_sqrshrn_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3348 def int_aarch64_sve_uqrshrn_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3349 def int_aarch64_sve_sqrshrn_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3350 def int_aarch64_sve_uqrshrn_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3351 3352 def int_aarch64_sve_sqrshru_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3353 def int_aarch64_sve_sqrshru_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3354 3355 def int_aarch64_sve_sqrshrun_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3356 def int_aarch64_sve_sqrshrun_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3357 3358 // 3359 // Multi-vector multiply-add/subtract long 3360 // 3361 3362 foreach ty = ["f", "s", "u"] in { 3363 foreach instr = ["mlal", "mlsl"] in { 3364 def int_aarch64_sme_ # ty # instr # _single_vg2x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; 3365 def int_aarch64_sme_ # ty # instr # _single_vg2x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3366 def int_aarch64_sme_ # ty # instr # _single_vg2x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3367 3368 def int_aarch64_sme_ # ty # instr # _vg2x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3369 def int_aarch64_sme_ # ty # instr # _vg2x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3370 3371 def int_aarch64_sme_ # ty # instr # _lane_vg2x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3372 def int_aarch64_sme_ # ty # instr # _lane_vg2x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3373 def int_aarch64_sme_ # ty # instr # _lane_vg2x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3374 } 3375 } 3376 3377 // 3378 // Multi-vector multiply-add long long 3379 // 3380 3381 foreach ty = ["s", "u"] in { 3382 foreach instr = ["mla", "mls"] in { 3383 foreach za = ["za32", "za64"] in { 3384 def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; 3385 def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3386 def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3387 3388 def int_aarch64_sme_ # ty # instr # _ # za # _vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3389 def int_aarch64_sme_ # ty # instr # _ # za # _vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3390 3391 def int_aarch64_sme_ # ty # instr # _ # za # _lane_vg4x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3392 def int_aarch64_sme_ # ty # instr # _ # za # _lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3393 def int_aarch64_sme_ # ty # instr # _ # za # _lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3394 } 3395 } 3396 } 3397 3398 def int_aarch64_sme_sumla_za32_single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3399 def int_aarch64_sme_sumla_za32_single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3400 3401 def int_aarch64_sme_sumla_za32_lane_vg4x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3402 def int_aarch64_sme_sumla_za32_lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3403 def int_aarch64_sme_sumla_za32_lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3404 3405 def int_aarch64_sme_usmla_za32_single_vg4x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; 3406 def int_aarch64_sme_usmla_za32_single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3407 def int_aarch64_sme_usmla_za32_single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3408 3409 def int_aarch64_sme_usmla_za32_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3410 def int_aarch64_sme_usmla_za32_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3411 3412 def int_aarch64_sme_usmla_za32_lane_vg4x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3413 def int_aarch64_sme_usmla_za32_lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3414 def int_aarch64_sme_usmla_za32_lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3415 3416 def int_aarch64_sve_bfmlslb : SME2_BFMLS_Intrinsic; 3417 def int_aarch64_sve_bfmlslb_lane : SME2_BFMLS_Lane_Intrinsic; 3418 3419 def int_aarch64_sve_bfmlslt : SME2_BFMLS_Intrinsic; 3420 def int_aarch64_sve_bfmlslt_lane : SME2_BFMLS_Lane_Intrinsic; 3421 3422 // Multi-vector zeroing 3423 3424 foreach vg = ["vg1x2", "vg1x4", "vg2x1", "vg2x2", "vg2x4", "vg4x1", "vg4x2", "vg4x4"] in { 3425 def int_aarch64_sme_zero_za64_ # vg : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects]>; 3426 } 3427 3428 // Multi-vector signed saturating doubling multiply high 3429 3430 def int_aarch64_sve_sqdmulh_single_vgx2 : SME2_VG2_Multi_Single_Intrinsic; 3431 def int_aarch64_sve_sqdmulh_single_vgx4 : SME2_VG4_Multi_Single_Intrinsic; 3432 3433 def int_aarch64_sve_sqdmulh_vgx2 : SME2_VG2_Multi_Multi_Intrinsic; 3434 def int_aarch64_sve_sqdmulh_vgx4 : SME2_VG4_Multi_Multi_Intrinsic; 3435 3436 // Multi-vector floating-point round to integral value 3437 3438 foreach inst = ["a", "m", "n", "p"] in { 3439 def int_aarch64_sve_frint # inst # _x2 : SVE2_VG2_ZipUzp_Intrinsic; 3440 def int_aarch64_sve_frint # inst # _x4 : SVE2_VG4_ZipUzp_Intrinsic; 3441 } 3442 3443 // 3444 // Multi-vector min/max 3445 // 3446 3447 foreach ty = ["f", "s", "u"] in { 3448 foreach instr = ["max", "min"] in { 3449 def int_aarch64_sve_ # ty # instr # _single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3450 def int_aarch64_sve_ # ty # instr # _single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3451 3452 def int_aarch64_sve_ # ty # instr # _x2 : SME2_VG2_Multi_Multi_Intrinsic; 3453 def int_aarch64_sve_ # ty # instr # _x4 : SME2_VG4_Multi_Multi_Intrinsic; 3454 } 3455 } 3456 3457 // 3458 // Multi-vector floating point min/max number 3459 // 3460 3461 foreach instr = ["fmaxnm", "fminnm"] in { 3462 def int_aarch64_sve_ # instr # _single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3463 def int_aarch64_sve_ # instr # _single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3464 3465 def int_aarch64_sve_ # instr # _x2 : SME2_VG2_Multi_Multi_Intrinsic; 3466 def int_aarch64_sve_ # instr # _x4 : SME2_VG4_Multi_Multi_Intrinsic; 3467 } 3468 3469 // 3470 // Multi-vector vertical dot-products 3471 // 3472 3473 def int_aarch64_sme_fvdot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3474 3475 foreach ty = ["s", "u"] in { 3476 def int_aarch64_sme_ #ty # vdot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3477 def int_aarch64_sme_ #ty # vdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3478 def int_aarch64_sme_ #ty # vdot_lane_za64_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3479 } 3480 3481 def int_aarch64_sme_suvdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3482 def int_aarch64_sme_usvdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3483 3484 3485 // 3486 //Multi-vector floating-point convert from half-precision to deinterleaved single-precision. 3487 // 3488 3489 def int_aarch64_sve_fcvtl_widen_x2 : SME2_CVT_WIDENING_VG2_Intrinsic; 3490 3491 // 3492 // Multi-vector floating-point CVT from single-precision to interleaved half-precision/BFloat16 3493 // 3494 def int_aarch64_sve_fcvtn_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3495 def int_aarch64_sve_bfcvtn_x2 : SME2_CVT_VG2_SINGLE_BF16_Intrinsic; 3496 3497 // 3498 // Multi-vector convert to/from floating-point. 3499 // 3500 def int_aarch64_sve_fcvt_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3501 def int_aarch64_sve_bfcvt_x2 : SME2_CVT_VG2_SINGLE_BF16_Intrinsic; 3502 def int_aarch64_sve_fcvtzs_x2 : SME2_CVT_X2_Intrinsic; 3503 def int_aarch64_sve_fcvtzu_x2 : SME2_CVT_X2_Intrinsic; 3504 def int_aarch64_sve_scvtf_x2 : SME2_CVT_X2_Intrinsic; 3505 def int_aarch64_sve_ucvtf_x2 : SME2_CVT_X2_Intrinsic; 3506 def int_aarch64_sve_fcvtzs_x4 : SME2_CVT_X4_Intrinsic; 3507 def int_aarch64_sve_fcvtzu_x4 : SME2_CVT_X4_Intrinsic; 3508 def int_aarch64_sve_scvtf_x4 : SME2_CVT_X4_Intrinsic; 3509 def int_aarch64_sve_ucvtf_x4 : SME2_CVT_X4_Intrinsic; 3510 def int_aarch64_sve_fcvt_widen_x2 : SME2_CVT_WIDENING_VG2_Intrinsic; 3511 // 3512 // Multi-vector saturating extract narrow 3513 // 3514 def int_aarch64_sve_sqcvt_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3515 def int_aarch64_sve_uqcvt_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3516 def int_aarch64_sve_sqcvtu_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3517 def int_aarch64_sve_sqcvt_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3518 def int_aarch64_sve_uqcvt_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3519 def int_aarch64_sve_sqcvtu_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3520 3521 // 3522 // Multi-vector saturating extract narrow and interleave 3523 // 3524 def int_aarch64_sve_sqcvtn_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3525 def int_aarch64_sve_uqcvtn_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3526 def int_aarch64_sve_sqcvtun_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3527 def int_aarch64_sve_sqcvtn_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3528 def int_aarch64_sve_uqcvtn_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3529 def int_aarch64_sve_sqcvtun_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3530 3531 // 3532 // Multi-Single add/sub 3533 // 3534 def int_aarch64_sme_add_write_single_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3535 def int_aarch64_sme_sub_write_single_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3536 def int_aarch64_sme_add_write_single_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3537 def int_aarch64_sme_sub_write_single_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3538 3539 // 3540 // Multi-Multi add/sub 3541 // 3542 def int_aarch64_sme_add_write_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3543 def int_aarch64_sme_sub_write_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3544 def int_aarch64_sme_add_write_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3545 def int_aarch64_sme_sub_write_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3546 3547 // Multi-vector clamps 3548 def int_aarch64_sve_sclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3549 def int_aarch64_sve_uclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3550 def int_aarch64_sve_fclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3551 def int_aarch64_sve_bfclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3552 3553 def int_aarch64_sve_sclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3554 def int_aarch64_sve_uclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3555 def int_aarch64_sve_fclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3556 def int_aarch64_sve_bfclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3557 3558 // 3559 // Multi-vector add/sub and accumulate into ZA 3560 // 3561 foreach intr = ["add", "sub"] in { 3562 foreach za = ["za16","za32", "za64"] in { 3563 def int_aarch64_sme_ # intr # _ # za # _vg1x2 : SME2_ZA_Write_VG2_Intrinsic; 3564 def int_aarch64_sme_ # intr # _ # za # _vg1x4 : SME2_ZA_Write_VG4_Intrinsic; 3565 } 3566 } 3567 3568 // 3569 // Move multi-vectors to/from ZA 3570 // 3571 3572 def int_aarch64_sme_read_hor_vg2 : SME2_Matrix_TileVector_Read_VG2_Intrinsic; 3573 def int_aarch64_sme_read_hor_vg4 : SME2_Matrix_TileVector_Read_VG4_Intrinsic; 3574 3575 def int_aarch64_sme_read_ver_vg2 : SME2_Matrix_TileVector_Read_VG2_Intrinsic; 3576 def int_aarch64_sme_read_ver_vg4 : SME2_Matrix_TileVector_Read_VG4_Intrinsic; 3577 3578 def int_aarch64_sme_read_vg1x2 : SME2_ZA_ArrayVector_Read_VG2_Intrinsic; 3579 def int_aarch64_sme_read_vg1x4 : SME2_ZA_ArrayVector_Read_VG4_Intrinsic; 3580 3581 def int_aarch64_sme_write_hor_vg2 : SME2_Matrix_TileVector_Write_VG2_Intrinsic; 3582 def int_aarch64_sme_write_hor_vg4 : SME2_Matrix_TileVector_Write_VG4_Intrinsic; 3583 3584 def int_aarch64_sme_write_ver_vg2 : SME2_Matrix_TileVector_Write_VG2_Intrinsic; 3585 def int_aarch64_sme_write_ver_vg4 : SME2_Matrix_TileVector_Write_VG4_Intrinsic; 3586 3587 def int_aarch64_sme_write_vg1x2 : SME2_ZA_ArrayVector_Write_VG2_Intrinsic; 3588 def int_aarch64_sme_write_vg1x4 : SME2_ZA_ArrayVector_Write_VG4_Intrinsic; 3589 3590 // 3591 // Multi-Single Vector add 3592 // 3593 def int_aarch64_sve_add_single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3594 def int_aarch64_sve_add_single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3595 3596 // 2-way and 4-way multi-vector signed/unsigned integer dot-product 3597 foreach ty = ["s", "u"] in { 3598 foreach sz = ["za32", "za64"] in { 3599 def int_aarch64_sme_ # ty # dot_single_ # sz # _vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3600 def int_aarch64_sme_ # ty # dot_single_ # sz # _vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3601 3602 def int_aarch64_sme_ # ty # dot_ # sz # _vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3603 def int_aarch64_sme_ # ty # dot_ # sz # _vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3604 3605 def int_aarch64_sme_ # ty # dot_lane_ # sz # _vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3606 def int_aarch64_sme_ # ty # dot_lane_ # sz # _vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3607 } 3608 } 3609 3610 foreach ty = ["su", "us"] in { 3611 def int_aarch64_sme_ # ty # dot_single_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3612 def int_aarch64_sme_ # ty # dot_single_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3613 3614 def int_aarch64_sme_ # ty # dot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3615 def int_aarch64_sme_ # ty # dot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3616 } 3617 3618 def int_aarch64_sme_usdot_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3619 def int_aarch64_sme_usdot_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3620 3621 // Multi-vector half-precision or bfloat floating-point dot-product 3622 def int_aarch64_sme_fdot_single_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3623 def int_aarch64_sme_fdot_single_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3624 3625 def int_aarch64_sme_fdot_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3626 def int_aarch64_sme_fdot_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3627 3628 def int_aarch64_sme_fdot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3629 def int_aarch64_sme_fdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3630 3631 // Multi-vector zip and unzips 3632 def int_aarch64_sve_zip_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3633 def int_aarch64_sve_zipq_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3634 def int_aarch64_sve_zip_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3635 def int_aarch64_sve_zipq_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3636 def int_aarch64_sve_uzp_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3637 def int_aarch64_sve_uzpq_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3638 def int_aarch64_sve_uzp_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3639 def int_aarch64_sve_uzpq_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3640 3641 // Vector dot-products (2-way) 3642 def int_aarch64_sve_sdot_x2 : SVE2_3VectorArg_Long_Intrinsic; 3643 def int_aarch64_sve_udot_x2 : SVE2_3VectorArg_Long_Intrinsic; 3644 def int_aarch64_sve_fdot_x2 : SVE2_3VectorArg_Long_Intrinsic; 3645 def int_aarch64_sve_sdot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic; 3646 def int_aarch64_sve_udot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic; 3647 def int_aarch64_sve_fdot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic; 3648 3649 // 3650 // Signed/unsigned multi-vector unpacks 3651 // 3652 def int_aarch64_sve_sunpk_x2 : SME2_VG2_Unpk_Intrinsic; 3653 def int_aarch64_sve_uunpk_x2 : SME2_VG2_Unpk_Intrinsic; 3654 def int_aarch64_sve_sunpk_x4 : SME2_VG4_Unpk_Intrinsic; 3655 def int_aarch64_sve_uunpk_x4 : SME2_VG4_Unpk_Intrinsic; 3656 3657 // 2-way and 4-way vector selects 3658 def int_aarch64_sve_sel_x2 : SVE2_VG2_Sel_Intrinsic; 3659 def int_aarch64_sve_sel_x4 : SVE2_VG4_Sel_Intrinsic; 3660 3661 class SME_LDR_STR_ZT_Intrinsic 3662 : DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty]>; 3663 def int_aarch64_sme_ldr_zt : SME_LDR_STR_ZT_Intrinsic; 3664 def int_aarch64_sme_str_zt : SME_LDR_STR_ZT_Intrinsic; 3665 3666 // 3667 // Zero ZT0 3668 // 3669 def int_aarch64_sme_zero_zt : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrWriteMem]>; 3670 3671 // 3672 // Lookup table expand one register 3673 // 3674 def int_aarch64_sme_luti2_lane_zt 3675 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3676 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3677 def int_aarch64_sme_luti4_lane_zt 3678 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3679 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3680 3681 // Lookup table expand two registers 3682 // 3683 def int_aarch64_sme_luti2_lane_zt_x2 3684 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3685 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3686 def int_aarch64_sme_luti4_lane_zt_x2 3687 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3688 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3689 3690 // 3691 // Lookup table expand four registers 3692 // 3693 def int_aarch64_sme_luti2_lane_zt_x4 3694 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3695 [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3696 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3697 def int_aarch64_sme_luti4_lane_zt_x4 3698 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3699 [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3700 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3701} 3702 3703// SVE2.1 - ZIPQ1, ZIPQ2, UZPQ1, UZPQ2 3704// 3705def int_aarch64_sve_zipq1 : AdvSIMD_2VectorArg_Intrinsic; 3706def int_aarch64_sve_zipq2 : AdvSIMD_2VectorArg_Intrinsic; 3707def int_aarch64_sve_uzpq1 : AdvSIMD_2VectorArg_Intrinsic; 3708def int_aarch64_sve_uzpq2 : AdvSIMD_2VectorArg_Intrinsic; 3709 3710// SVE2.1 - Programmable table lookup within each quadword vector segment 3711// (zeroing)/(merging) 3712// 3713def int_aarch64_sve_tblq : AdvSIMD_SVE_TBL_Intrinsic; 3714def int_aarch64_sve_tbxq : AdvSIMD_SVE2_TBX_Intrinsic; 3715 3716// SVE2.1 - Extract vector segment from each pair of quadword segments. 3717// 3718def int_aarch64_sve_extq : AdvSIMD_2VectorArgIndexed_Intrinsic; 3719 3720// 3721// SVE2.1 - Move predicate to/from vector 3722// 3723def int_aarch64_sve_pmov_to_pred_lane : SVE2_1VectorArgIndexed_Pred_Intrinsic; 3724 3725def int_aarch64_sve_pmov_to_pred_lane_zero : SVE2_1VectorArg_Pred_Intrinsic; 3726 3727def int_aarch64_sve_pmov_to_vector_lane_merging : SVE2_Pred_1VectorArgIndexed_Intrinsic; 3728def int_aarch64_sve_pmov_to_vector_lane_zeroing : SVE2_Pred_1VectorArg_Intrinsic; 3729 3730def int_aarch64_sme_mopa_nonwide : SME_OuterProduct_Intrinsic; 3731def int_aarch64_sme_mops_nonwide : SME_OuterProduct_Intrinsic; 3732 3733