xref: /freebsd/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsAArch64.td (revision acb1f1269c6f4ff89a0d28ba742f6687e9ef779d)
1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines all of the AARCH64-specific intrinsics.
10//
11//===----------------------------------------------------------------------===//
12
13let TargetPrefix = "aarch64" in {
14
15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
16                                 [IntrNoFree, IntrWillReturn]>;
17def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
18                                  [IntrNoFree, IntrWillReturn]>;
19def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
20                                 [IntrNoFree, IntrWillReturn]>;
21def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
22                                  [IntrNoFree, IntrWillReturn]>;
23
24def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
25                                 [IntrNoFree, IntrWillReturn]>;
26def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
27                                  [IntrNoFree, IntrWillReturn]>;
28def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
29                               [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
30                               [IntrNoFree, IntrWillReturn]>;
31def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
32                                  [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
33                                  [IntrNoFree, IntrWillReturn]>;
34
35def int_aarch64_clrex : Intrinsic<[]>;
36
37def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
38                                LLVMMatchType<0>], [IntrNoMem]>;
39def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
40                                LLVMMatchType<0>], [IntrNoMem]>;
41
42def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
43
44def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
45def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
46
47//===----------------------------------------------------------------------===//
48// HINT
49
50def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;
51
52//===----------------------------------------------------------------------===//
53// Data Barrier Instructions
54
55def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">,
56                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
57def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">,
58                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
59def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">,
60                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
61
62// A space-consuming intrinsic primarily for testing block and jump table
63// placements. The first argument is the number of bytes this "instruction"
64// takes up, the second and return value are essentially chains, used to force
65// ordering during ISel.
66def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
67
68}
69
70//===----------------------------------------------------------------------===//
71// Advanced SIMD (NEON)
72
73let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
74  class AdvSIMD_2Scalar_Float_Intrinsic
75    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
76                [IntrNoMem]>;
77
78  class AdvSIMD_FPToIntRounding_Intrinsic
79    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
80
81  class AdvSIMD_1IntArg_Intrinsic
82    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
83  class AdvSIMD_1FloatArg_Intrinsic
84    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
85  class AdvSIMD_1VectorArg_Intrinsic
86    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
87  class AdvSIMD_1VectorArg_Expand_Intrinsic
88    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
89  class AdvSIMD_1VectorArg_Long_Intrinsic
90    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
91  class AdvSIMD_1IntArg_Narrow_Intrinsic
92    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
93  class AdvSIMD_1VectorArg_Narrow_Intrinsic
94    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
95  class AdvSIMD_1VectorArg_Int_Across_Intrinsic
96    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
97  class AdvSIMD_1VectorArg_Float_Across_Intrinsic
98    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
99
100  class AdvSIMD_2IntArg_Intrinsic
101    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
102                [IntrNoMem]>;
103  class AdvSIMD_2FloatArg_Intrinsic
104    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
105                [IntrNoMem]>;
106  class AdvSIMD_2VectorArg_Intrinsic
107    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
108                [IntrNoMem]>;
109  class AdvSIMD_2VectorArg_Compare_Intrinsic
110    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
111                [IntrNoMem]>;
112  class AdvSIMD_2Arg_FloatCompare_Intrinsic
113    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
114                [IntrNoMem]>;
115  class AdvSIMD_2VectorArg_Long_Intrinsic
116    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
117                [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
118                [IntrNoMem]>;
119  class AdvSIMD_2VectorArg_Wide_Intrinsic
120    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
121                [LLVMMatchType<0>, LLVMTruncatedType<0>],
122                [IntrNoMem]>;
123  class AdvSIMD_2VectorArg_Narrow_Intrinsic
124    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
125                [LLVMExtendedType<0>, LLVMExtendedType<0>],
126                [IntrNoMem]>;
127  class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
128    : DefaultAttrsIntrinsic<[llvm_anyint_ty],
129                [LLVMExtendedType<0>, llvm_i32_ty],
130                [IntrNoMem]>;
131  class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
132    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
133                [llvm_anyvector_ty],
134                [IntrNoMem]>;
135  class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
136    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
137                [LLVMTruncatedType<0>],
138                [IntrNoMem]>;
139  class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
140    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
141                [LLVMTruncatedType<0>, llvm_i32_ty],
142                [IntrNoMem]>;
143  class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
144    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
145                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
146                [IntrNoMem]>;
147  class AdvSIMD_2VectorArg_Lane_Intrinsic
148    : DefaultAttrsIntrinsic<[llvm_anyint_ty],
149                [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
150                [IntrNoMem]>;
151
152  class AdvSIMD_3VectorArg_Intrinsic
153      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
154               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
155               [IntrNoMem]>;
156  class AdvSIMD_3VectorArg_Scalar_Intrinsic
157      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
158               [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
159               [IntrNoMem]>;
160  class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
161      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
162               [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
163                LLVMMatchType<1>], [IntrNoMem]>;
164  class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
165    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
166                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
167                [IntrNoMem]>;
168  class AdvSIMD_CvtFxToFP_Intrinsic
169    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
170                [IntrNoMem]>;
171  class AdvSIMD_CvtFPToFx_Intrinsic
172    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
173                [IntrNoMem]>;
174
175  class AdvSIMD_1Arg_Intrinsic
176    : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
177
178  class AdvSIMD_Dot_Intrinsic
179    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
180                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
181                [IntrNoMem]>;
182
183  class AdvSIMD_FP16FML_Intrinsic
184    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
185                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
186                [IntrNoMem]>;
187
188  class AdvSIMD_MatMul_Intrinsic
189    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
190                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
191                [IntrNoMem]>;
192
193  class AdvSIMD_FML_Intrinsic
194    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
195                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
196                [IntrNoMem]>;
197
198  class AdvSIMD_BF16FML_Intrinsic
199    : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
200                [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
201                [IntrNoMem]>;
202}
203
204// Arithmetic ops
205
206let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
207  // Vector Add Across Lanes
208  def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
209  def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
210  def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
211
212  // Vector Long Add Across Lanes
213  def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
214  def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
215
216  // Vector Halving Add
217  def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
218  def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
219
220  // Vector Rounding Halving Add
221  def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
222  def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
223
224  // Vector Saturating Add
225  def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
226  def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
227  def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
228  def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
229
230  // Vector Add High-Half
231  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
232  // header is no longer supported.
233  def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
234
235  // Vector Rounding Add High-Half
236  def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
237
238  // Vector Saturating Doubling Multiply High
239  def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
240  def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
241  def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
242
243  // Vector Saturating Rounding Doubling Multiply High
244  def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
245  def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
246  def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
247
248  // Vector Polynominal Multiply
249  def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
250
251  // Vector Long Multiply
252  def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
253  def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
254  def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
255
256  // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
257  // it with a v16i8.
258  def int_aarch64_neon_pmull64 :
259        DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
260
261  // Vector Extending Multiply
262  def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
263    let IntrProperties = [IntrNoMem, Commutative];
264  }
265
266  // Vector Saturating Doubling Long Multiply
267  def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
268  def int_aarch64_neon_sqdmulls_scalar
269    : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
270
271  // Vector Halving Subtract
272  def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
273  def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
274
275  // Vector Saturating Subtract
276  def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
277  def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
278
279  // Vector Subtract High-Half
280  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
281  // header is no longer supported.
282  def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
283
284  // Vector Rounding Subtract High-Half
285  def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
286
287  // Vector Compare Absolute Greater-than-or-equal
288  def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
289
290  // Vector Compare Absolute Greater-than
291  def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
292
293  // Vector Absolute Difference
294  def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
295  def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
296  def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
297
298  // Scalar Absolute Difference
299  def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
300
301  // Vector Max
302  def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
303  def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
304  def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
305  def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
306
307  // Vector Max Across Lanes
308  def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
309  def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
310  def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
311  def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
312
313  // Vector Min
314  def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
315  def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
316  def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
317  def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
318
319  // Vector Min/Max Number
320  def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
321  def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
322
323  // Vector Min Across Lanes
324  def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
325  def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
326  def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
327  def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
328
329  // Pairwise Add
330  def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
331  def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
332
333  // Long Pairwise Add
334  // FIXME: In theory, we shouldn't need intrinsics for saddlp or
335  // uaddlp, but tblgen's type inference currently can't handle the
336  // pattern fragments this ends up generating.
337  def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
338  def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
339
340  // Folding Maximum
341  def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
342  def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
343  def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
344
345  // Folding Minimum
346  def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
347  def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
348  def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
349
350  // Reciprocal Estimate/Step
351  def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
352  def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
353
354  // Reciprocal Exponent
355  def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
356
357  // Vector Saturating Shift Left
358  def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
359  def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
360
361  // Vector Rounding Shift Left
362  def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
363  def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
364
365  // Vector Saturating Rounding Shift Left
366  def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
367  def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
368
369  // Vector Signed->Unsigned Shift Left by Constant
370  def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
371
372  // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
373  def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
374
375  // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
376  def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
377
378  // Vector Narrowing Shift Right by Constant
379  def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
380  def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
381
382  // Vector Rounding Narrowing Shift Right by Constant
383  def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
384
385  // Vector Rounding Narrowing Saturating Shift Right by Constant
386  def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
387  def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
388
389  // Vector Shift Left
390  def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
391  def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
392
393  // Vector Widening Shift Left by Constant
394  def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
395  def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
396  def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
397
398  // Vector Shift Right by Constant and Insert
399  def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
400
401  // Vector Shift Left by Constant and Insert
402  def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
403
404  // Vector Saturating Narrow
405  def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
406  def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
407  def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
408  def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
409
410  // Vector Saturating Extract and Unsigned Narrow
411  def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
412  def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
413
414  // Vector Absolute Value
415  def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
416
417  // Vector Saturating Absolute Value
418  def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
419
420  // Vector Saturating Negation
421  def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
422
423  // Vector Count Leading Sign Bits
424  def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
425
426  // Vector Reciprocal Estimate
427  def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
428  def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
429
430  // Vector Square Root Estimate
431  def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
432  def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
433
434  // Vector Bitwise Reverse
435  def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
436
437  // Vector Conversions Between Half-Precision and Single-Precision.
438  def int_aarch64_neon_vcvtfp2hf
439    : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
440  def int_aarch64_neon_vcvthf2fp
441    : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
442
443  // Vector Conversions Between Floating-point and Fixed-point.
444  def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
445  def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
446  def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
447  def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
448
449  // Vector FP->Int Conversions
450  def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
451  def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
452  def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
453  def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
454  def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
455  def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
456  def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
457  def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
458  def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
459  def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
460
461  // Vector FP Rounding: only ties to even is unrepresented by a normal
462  // intrinsic.
463  def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
464
465  // Scalar FP->Int conversions
466
467  // Vector FP Inexact Narrowing
468  def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
469
470  // Scalar FP Inexact Narrowing
471  def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty],
472                                        [IntrNoMem]>;
473
474  // v8.2-A Dot Product
475  def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
476  def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
477
478// v8.6-A Matrix Multiply Intrinsics
479  def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
480  def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
481  def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
482  def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
483  def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
484  def int_aarch64_neon_bfmmla
485    : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
486                [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
487                [IntrNoMem]>;
488  def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic;
489  def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic;
490
491
492  // v8.6-A Bfloat Intrinsics
493  def int_aarch64_neon_bfcvt
494    : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
495  def int_aarch64_neon_bfcvtn
496    : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
497  def int_aarch64_neon_bfcvtn2
498    : DefaultAttrsIntrinsic<[llvm_v8bf16_ty],
499                [llvm_v8bf16_ty, llvm_v4f32_ty],
500                [IntrNoMem]>;
501
502  // v8.2-A FP16 Fused Multiply-Add Long
503  def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
504  def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
505  def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
506  def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
507
508  // v8.3-A Floating-point complex add
509  def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
510  def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
511
512  def int_aarch64_neon_vcmla_rot0   : AdvSIMD_3VectorArg_Intrinsic;
513  def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;
514  def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;
515  def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;
516}
517
518let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
519  class AdvSIMD_2Vector2Index_Intrinsic
520    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
521                [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
522                [IntrNoMem]>;
523}
524
525// Vector element to element moves
526def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
527
528let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
529  class AdvSIMD_1Vec_Load_Intrinsic
530      : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
531                  [IntrReadMem, IntrArgMemOnly]>;
532  class AdvSIMD_1Vec_Store_Lane_Intrinsic
533    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
534                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
535
536  class AdvSIMD_2Vec_Load_Intrinsic
537    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
538                [LLVMAnyPointerType<LLVMMatchType<0>>],
539                [IntrReadMem, IntrArgMemOnly]>;
540  class AdvSIMD_2Vec_Load_Lane_Intrinsic
541    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
542                [LLVMMatchType<0>, llvm_anyvector_ty,
543                 llvm_i64_ty, llvm_anyptr_ty],
544                [IntrReadMem, IntrArgMemOnly]>;
545  class AdvSIMD_2Vec_Store_Intrinsic
546    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
547                     LLVMAnyPointerType<LLVMMatchType<0>>],
548                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
549  class AdvSIMD_2Vec_Store_Lane_Intrinsic
550    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
551                 llvm_i64_ty, llvm_anyptr_ty],
552                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
553
554  class AdvSIMD_3Vec_Load_Intrinsic
555    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
556                [LLVMAnyPointerType<LLVMMatchType<0>>],
557                [IntrReadMem, IntrArgMemOnly]>;
558  class AdvSIMD_3Vec_Load_Lane_Intrinsic
559    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
560                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
561                 llvm_i64_ty, llvm_anyptr_ty],
562                [IntrReadMem, IntrArgMemOnly]>;
563  class AdvSIMD_3Vec_Store_Intrinsic
564    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
565                     LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
566                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
567  class AdvSIMD_3Vec_Store_Lane_Intrinsic
568    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,
569                 LLVMMatchType<0>, LLVMMatchType<0>,
570                 llvm_i64_ty, llvm_anyptr_ty],
571                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
572
573  class AdvSIMD_4Vec_Load_Intrinsic
574    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
575                 LLVMMatchType<0>, llvm_anyvector_ty],
576                [LLVMAnyPointerType<LLVMMatchType<0>>],
577                [IntrReadMem, IntrArgMemOnly]>;
578  class AdvSIMD_4Vec_Load_Lane_Intrinsic
579    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
580                 LLVMMatchType<0>, LLVMMatchType<0>],
581                [LLVMMatchType<0>, LLVMMatchType<0>,
582                 LLVMMatchType<0>, llvm_anyvector_ty,
583                 llvm_i64_ty, llvm_anyptr_ty],
584                [IntrReadMem, IntrArgMemOnly]>;
585  class AdvSIMD_4Vec_Store_Intrinsic
586    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
587                 LLVMMatchType<0>, LLVMMatchType<0>,
588                 LLVMAnyPointerType<LLVMMatchType<0>>],
589                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
590  class AdvSIMD_4Vec_Store_Lane_Intrinsic
591    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
592                 LLVMMatchType<0>, LLVMMatchType<0>,
593                 llvm_i64_ty, llvm_anyptr_ty],
594                [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
595}
596
597// Memory ops
598
599def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
600def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
601def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
602
603def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
604def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
605def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
606
607def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
608def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
609def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
610
611def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
612def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
613def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
614
615def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
616def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
617def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
618
619def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
620def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
621def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
622
623def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
624def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
625def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
626
627let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
628  class AdvSIMD_Tbl1_Intrinsic
629    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
630                [IntrNoMem]>;
631  class AdvSIMD_Tbl2_Intrinsic
632    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
633                [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
634  class AdvSIMD_Tbl3_Intrinsic
635    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
636                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
637                 LLVMMatchType<0>],
638                [IntrNoMem]>;
639  class AdvSIMD_Tbl4_Intrinsic
640    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
641                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
642                 LLVMMatchType<0>],
643                [IntrNoMem]>;
644
645  class AdvSIMD_Tbx1_Intrinsic
646    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
647                [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
648                [IntrNoMem]>;
649  class AdvSIMD_Tbx2_Intrinsic
650    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
651                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
652                 LLVMMatchType<0>],
653                [IntrNoMem]>;
654  class AdvSIMD_Tbx3_Intrinsic
655    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
656                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
657                 llvm_v16i8_ty, LLVMMatchType<0>],
658                [IntrNoMem]>;
659  class AdvSIMD_Tbx4_Intrinsic
660    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
661                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
662                 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
663                [IntrNoMem]>;
664}
665def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
666def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
667def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
668def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
669
670def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
671def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
672def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
673def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
674
675let TargetPrefix = "aarch64" in {
676  class FPCR_Get_Intrinsic
677    : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
678}
679
680// FPCR
681def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
682
683let TargetPrefix = "aarch64" in {
684  class Crypto_AES_DataKey_Intrinsic
685    : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
686
687  class Crypto_AES_Data_Intrinsic
688    : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
689
690  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
691  // (v4i32).
692  class Crypto_SHA_5Hash4Schedule_Intrinsic
693    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
694                [IntrNoMem]>;
695
696  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
697  // (v4i32).
698  class Crypto_SHA_1Hash_Intrinsic
699    : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
700
701  // SHA intrinsic taking 8 words of the schedule
702  class Crypto_SHA_8Schedule_Intrinsic
703    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
704
705  // SHA intrinsic taking 12 words of the schedule
706  class Crypto_SHA_12Schedule_Intrinsic
707    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
708                [IntrNoMem]>;
709
710  // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
711  class Crypto_SHA_8Hash4Schedule_Intrinsic
712    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
713                [IntrNoMem]>;
714}
715
716// AES
717def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
718def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
719def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
720def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
721
722// SHA1
723def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
724def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
725def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
726def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
727
728def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
729def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
730
731// SHA256
732def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
733def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
734def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
735def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
736
737//===----------------------------------------------------------------------===//
738// CRC32
739
740let TargetPrefix = "aarch64" in {
741
742def int_aarch64_crc32b  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
743    [IntrNoMem]>;
744def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
745    [IntrNoMem]>;
746def int_aarch64_crc32h  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
747    [IntrNoMem]>;
748def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
749    [IntrNoMem]>;
750def int_aarch64_crc32w  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
751    [IntrNoMem]>;
752def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
753    [IntrNoMem]>;
754def int_aarch64_crc32x  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
755    [IntrNoMem]>;
756def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
757    [IntrNoMem]>;
758}
759
760//===----------------------------------------------------------------------===//
761// Memory Tagging Extensions (MTE) Intrinsics
762let TargetPrefix = "aarch64" in {
763def int_aarch64_irg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
764    [IntrNoMem, IntrHasSideEffects]>;
765def int_aarch64_addg  : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
766    [IntrNoMem]>;
767def int_aarch64_gmi   : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
768    [IntrNoMem]>;
769def int_aarch64_ldg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
770    [IntrReadMem]>;
771def int_aarch64_stg   : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
772    [IntrWriteMem]>;
773def int_aarch64_subp :  DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
774    [IntrNoMem]>;
775
776// The following are codegen-only intrinsics for stack instrumentation.
777
778// Generate a randomly tagged stack base pointer.
779def int_aarch64_irg_sp   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty],
780    [IntrNoMem, IntrHasSideEffects]>;
781
782// Transfer pointer tag with offset.
783// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
784// * address is the address in ptr0
785// * tag is a function of (tag in baseptr, tag_offset).
786// ** Beware, this is not the same function as implemented by the ADDG instruction!
787//    Backend optimizations may change tag_offset; the only guarantee is that calls
788//    to tagp with the same pair of (baseptr, tag_offset) will produce pointers
789//    with the same tag value, assuming the set of excluded tags has not changed.
790// Address bits in baseptr and tag bits in ptr0 are ignored.
791// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
792//   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
793// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
794def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
795    [IntrNoMem, ImmArg<ArgIndex<2>>]>;
796
797// Update allocation tags for the memory range to match the tag in the pointer argument.
798def int_aarch64_settag  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
799    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
800
801// Update allocation tags for the memory range to match the tag in the pointer argument,
802// and set memory contents to zero.
803def int_aarch64_settag_zero  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
804    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
805
806// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
807def int_aarch64_stgp  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
808    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
809}
810
811// Transactional Memory Extension (TME) Intrinsics
812let TargetPrefix = "aarch64" in {
813def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
814                         Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;
815
816def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>;
817
818def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
819                          Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>;
820
821def int_aarch64_ttest   : GCCBuiltin<"__builtin_arm_ttest">,
822                          Intrinsic<[llvm_i64_ty], [],
823                                    [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;
824
825// Armv8.7-A load/store 64-byte intrinsics
826defvar data512 = !listsplat(llvm_i64_ty, 8);
827def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>;
828def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;
829def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
830def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
831
832}
833
834def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
835def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
836def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
837def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
838def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
839def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
840def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
841def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
842def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
843def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
844def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
845
846let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
847
848  class AdvSIMD_SVE_Create_2Vector_Tuple
849    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
850                [llvm_anyvector_ty, LLVMMatchType<1>],
851                [IntrReadMem]>;
852
853  class AdvSIMD_SVE_Create_3Vector_Tuple
854    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
855                [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
856                [IntrReadMem]>;
857
858  class AdvSIMD_SVE_Create_4Vector_Tuple
859    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
860                [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
861                 LLVMMatchType<1>],
862                [IntrReadMem]>;
863
864  class AdvSIMD_SVE_Set_Vector_Tuple
865    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
866                [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
867                [IntrReadMem, ImmArg<ArgIndex<1>>]>;
868
869  class AdvSIMD_SVE_Get_Vector_Tuple
870    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
871                [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
872
873  class AdvSIMD_ManyVec_PredLoad_Intrinsic
874    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
875                [IntrReadMem, IntrArgMemOnly]>;
876
877  class AdvSIMD_1Vec_PredLoad_Intrinsic
878    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
879                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
880                 LLVMPointerToElt<0>],
881                [IntrReadMem, IntrArgMemOnly]>;
882
883  class AdvSIMD_1Vec_PredStore_Intrinsic
884    : DefaultAttrsIntrinsic<[],
885                [llvm_anyvector_ty,
886                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
887                 LLVMPointerToElt<0>],
888                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
889
890  class AdvSIMD_2Vec_PredStore_Intrinsic
891      : DefaultAttrsIntrinsic<[],
892                  [llvm_anyvector_ty, LLVMMatchType<0>,
893                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
894                  [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
895
896  class AdvSIMD_3Vec_PredStore_Intrinsic
897      : DefaultAttrsIntrinsic<[],
898                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
899                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
900                  [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
901
902  class AdvSIMD_4Vec_PredStore_Intrinsic
903      : DefaultAttrsIntrinsic<[],
904                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
905                   LLVMMatchType<0>,
906                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
907                  [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
908
909  class AdvSIMD_SVE_Index_Intrinsic
910    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
911                [LLVMVectorElementType<0>,
912                 LLVMVectorElementType<0>],
913                [IntrNoMem]>;
914
915  class AdvSIMD_Merged1VectorArg_Intrinsic
916    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
917                [LLVMMatchType<0>,
918                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
919                 LLVMMatchType<0>],
920                [IntrNoMem]>;
921
922  class AdvSIMD_2VectorArgIndexed_Intrinsic
923    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
924                [LLVMMatchType<0>,
925                 LLVMMatchType<0>,
926                 llvm_i32_ty],
927                [IntrNoMem, ImmArg<ArgIndex<2>>]>;
928
929  class AdvSIMD_3VectorArgIndexed_Intrinsic
930    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
931                [LLVMMatchType<0>,
932                 LLVMMatchType<0>,
933                 LLVMMatchType<0>,
934                 llvm_i32_ty],
935                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
936
937  class AdvSIMD_Pred1VectorArg_Intrinsic
938    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
939                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
940                 LLVMMatchType<0>],
941                [IntrNoMem]>;
942
943  class AdvSIMD_Pred2VectorArg_Intrinsic
944    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
945                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
946                 LLVMMatchType<0>,
947                 LLVMMatchType<0>],
948                [IntrNoMem]>;
949
950  class AdvSIMD_Pred3VectorArg_Intrinsic
951    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
952                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
953                 LLVMMatchType<0>,
954                 LLVMMatchType<0>,
955                 LLVMMatchType<0>],
956                [IntrNoMem]>;
957
958  class AdvSIMD_SVE_Compare_Intrinsic
959    : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
960                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
961                 llvm_anyvector_ty,
962                 LLVMMatchType<0>],
963                [IntrNoMem]>;
964
965  class AdvSIMD_SVE_CompareWide_Intrinsic
966    : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
967                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
968                 llvm_anyvector_ty,
969                 llvm_nxv2i64_ty],
970                [IntrNoMem]>;
971
972  class AdvSIMD_SVE_Saturating_Intrinsic
973    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
974                [LLVMMatchType<0>,
975                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
976                [IntrNoMem]>;
977
978  class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
979    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
980                [LLVMMatchType<0>,
981                 llvm_i32_ty,
982                 llvm_i32_ty],
983                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
984
985  class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
986    : DefaultAttrsIntrinsic<[T],
987                [T, llvm_anyvector_ty],
988                [IntrNoMem]>;
989
990  class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
991    : DefaultAttrsIntrinsic<[T],
992                [T, llvm_i32_ty, llvm_i32_ty],
993                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
994
995  class AdvSIMD_SVE_CNT_Intrinsic
996    : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>],
997                [LLVMVectorOfBitcastsToInt<0>,
998                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
999                 llvm_anyvector_ty],
1000                [IntrNoMem]>;
1001
1002  class AdvSIMD_SVE_ReduceWithInit_Intrinsic
1003    : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
1004                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1005                 LLVMVectorElementType<0>,
1006                 llvm_anyvector_ty],
1007                [IntrNoMem]>;
1008
1009  class AdvSIMD_SVE_ShiftByImm_Intrinsic
1010    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1011                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1012                 LLVMMatchType<0>,
1013                 llvm_i32_ty],
1014                [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1015
1016  class AdvSIMD_SVE_ShiftWide_Intrinsic
1017    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1018                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1019                 LLVMMatchType<0>,
1020                 llvm_nxv2i64_ty],
1021                [IntrNoMem]>;
1022
1023  class AdvSIMD_SVE_Unpack_Intrinsic
1024    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1025               [LLVMSubdivide2VectorType<0>],
1026               [IntrNoMem]>;
1027
1028  class AdvSIMD_SVE_CADD_Intrinsic
1029    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1030                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1031                 LLVMMatchType<0>,
1032                 LLVMMatchType<0>,
1033                 llvm_i32_ty],
1034                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1035
1036  class AdvSIMD_SVE_CMLA_Intrinsic
1037    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1038                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1039                 LLVMMatchType<0>,
1040                 LLVMMatchType<0>,
1041                 LLVMMatchType<0>,
1042                 llvm_i32_ty],
1043                [IntrNoMem, ImmArg<ArgIndex<4>>]>;
1044
1045  class AdvSIMD_SVE_CMLA_LANE_Intrinsic
1046    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1047                [LLVMMatchType<0>,
1048                 LLVMMatchType<0>,
1049                 LLVMMatchType<0>,
1050                 llvm_i32_ty,
1051                 llvm_i32_ty],
1052                [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1053
1054  class AdvSIMD_SVE_DUP_Intrinsic
1055    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1056                [LLVMMatchType<0>,
1057                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1058                 LLVMVectorElementType<0>],
1059                [IntrNoMem]>;
1060
1061  class AdvSIMD_SVE_DUP_Unpred_Intrinsic
1062    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
1063                [IntrNoMem]>;
1064
1065  class AdvSIMD_SVE_DUPQ_Intrinsic
1066    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1067                [LLVMMatchType<0>,
1068                 llvm_i64_ty],
1069                [IntrNoMem]>;
1070
1071  class AdvSIMD_SVE_EXPA_Intrinsic
1072    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1073                [LLVMVectorOfBitcastsToInt<0>],
1074                [IntrNoMem]>;
1075
1076  class AdvSIMD_SVE_FCVT_Intrinsic
1077    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1078                [LLVMMatchType<0>,
1079                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1080                 llvm_anyvector_ty],
1081                [IntrNoMem]>;
1082
1083  class AdvSIMD_SVE_FCVTZS_Intrinsic
1084    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1085                [LLVMVectorOfBitcastsToInt<0>,
1086                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1087                 llvm_anyvector_ty],
1088                [IntrNoMem]>;
1089
1090  class AdvSIMD_SVE_INSR_Intrinsic
1091    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1092                [LLVMMatchType<0>,
1093                 LLVMVectorElementType<0>],
1094                [IntrNoMem]>;
1095
1096  class AdvSIMD_SVE_PTRUE_Intrinsic
1097    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1098                [llvm_i32_ty],
1099                [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1100
1101  class AdvSIMD_SVE_PUNPKHI_Intrinsic
1102    : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>],
1103                [llvm_anyvector_ty],
1104                [IntrNoMem]>;
1105
1106  class AdvSIMD_SVE_SCALE_Intrinsic
1107    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1108                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1109                 LLVMMatchType<0>,
1110                 LLVMVectorOfBitcastsToInt<0>],
1111                [IntrNoMem]>;
1112
1113  class AdvSIMD_SVE_SCVTF_Intrinsic
1114    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1115                [LLVMMatchType<0>,
1116                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1117                 llvm_anyvector_ty],
1118                [IntrNoMem]>;
1119
1120  class AdvSIMD_SVE_TSMUL_Intrinsic
1121    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1122                [LLVMMatchType<0>,
1123                 LLVMVectorOfBitcastsToInt<0>],
1124                [IntrNoMem]>;
1125
1126  class AdvSIMD_SVE_CNTB_Intrinsic
1127    : DefaultAttrsIntrinsic<[llvm_i64_ty],
1128                [llvm_i32_ty],
1129                [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1130
1131  class AdvSIMD_SVE_CNTP_Intrinsic
1132    : DefaultAttrsIntrinsic<[llvm_i64_ty],
1133                [llvm_anyvector_ty, LLVMMatchType<0>],
1134                [IntrNoMem]>;
1135
1136  class AdvSIMD_SVE_DOT_Intrinsic
1137    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1138                [LLVMMatchType<0>,
1139                 LLVMSubdivide4VectorType<0>,
1140                 LLVMSubdivide4VectorType<0>],
1141                [IntrNoMem]>;
1142
1143  class AdvSIMD_SVE_DOT_Indexed_Intrinsic
1144    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1145                [LLVMMatchType<0>,
1146                 LLVMSubdivide4VectorType<0>,
1147                 LLVMSubdivide4VectorType<0>,
1148                 llvm_i32_ty],
1149                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1150
1151  class AdvSIMD_SVE_PTEST_Intrinsic
1152    : DefaultAttrsIntrinsic<[llvm_i1_ty],
1153                [llvm_anyvector_ty,
1154                 LLVMMatchType<0>],
1155                [IntrNoMem]>;
1156
1157  class AdvSIMD_SVE_TBL_Intrinsic
1158    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1159                [LLVMMatchType<0>,
1160                 LLVMVectorOfBitcastsToInt<0>],
1161                [IntrNoMem]>;
1162
1163  class AdvSIMD_SVE2_TBX_Intrinsic
1164    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1165                [LLVMMatchType<0>,
1166                 LLVMMatchType<0>,
1167                 LLVMVectorOfBitcastsToInt<0>],
1168                [IntrNoMem]>;
1169
1170  class SVE2_1VectorArg_Long_Intrinsic
1171    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1172                [LLVMSubdivide2VectorType<0>,
1173                 llvm_i32_ty],
1174                [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1175
1176  class SVE2_2VectorArg_Long_Intrinsic
1177    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1178                [LLVMSubdivide2VectorType<0>,
1179                 LLVMSubdivide2VectorType<0>],
1180                [IntrNoMem]>;
1181
1182  class SVE2_2VectorArgIndexed_Long_Intrinsic
1183  : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1184              [LLVMSubdivide2VectorType<0>,
1185               LLVMSubdivide2VectorType<0>,
1186               llvm_i32_ty],
1187              [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1188
1189  class SVE2_2VectorArg_Wide_Intrinsic
1190    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1191                [LLVMMatchType<0>,
1192                 LLVMSubdivide2VectorType<0>],
1193                [IntrNoMem]>;
1194
1195  class SVE2_2VectorArg_Pred_Long_Intrinsic
1196    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1197                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1198                 LLVMMatchType<0>,
1199                 LLVMSubdivide2VectorType<0>],
1200                [IntrNoMem]>;
1201
1202  class SVE2_3VectorArg_Long_Intrinsic
1203    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1204                [LLVMMatchType<0>,
1205                 LLVMSubdivide2VectorType<0>,
1206                 LLVMSubdivide2VectorType<0>],
1207                [IntrNoMem]>;
1208
1209  class SVE2_3VectorArgIndexed_Long_Intrinsic
1210    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1211                [LLVMMatchType<0>,
1212                 LLVMSubdivide2VectorType<0>,
1213                 LLVMSubdivide2VectorType<0>,
1214                 llvm_i32_ty],
1215                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1216
1217  class SVE2_1VectorArg_Narrowing_Intrinsic
1218    : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1219                [llvm_anyvector_ty],
1220                [IntrNoMem]>;
1221
1222  class SVE2_Merged1VectorArg_Narrowing_Intrinsic
1223    : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1224                [LLVMSubdivide2VectorType<0>,
1225                 llvm_anyvector_ty],
1226                [IntrNoMem]>;
1227  class SVE2_2VectorArg_Narrowing_Intrinsic
1228      : DefaultAttrsIntrinsic<
1229            [LLVMSubdivide2VectorType<0>],
1230            [llvm_anyvector_ty, LLVMMatchType<0>],
1231            [IntrNoMem]>;
1232
1233  class SVE2_Merged2VectorArg_Narrowing_Intrinsic
1234      : DefaultAttrsIntrinsic<
1235            [LLVMSubdivide2VectorType<0>],
1236            [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
1237            [IntrNoMem]>;
1238
1239  class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
1240      : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1241                  [llvm_anyvector_ty, llvm_i32_ty],
1242                  [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1243
1244  class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
1245      : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
1246                  [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
1247                   llvm_i32_ty],
1248                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1249
1250  class SVE2_CONFLICT_DETECT_Intrinsic
1251    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1252                [LLVMAnyPointerType<llvm_any_ty>,
1253                 LLVMMatchType<1>]>;
1254
1255  class SVE2_3VectorArg_Indexed_Intrinsic
1256    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1257                [LLVMMatchType<0>,
1258                 LLVMSubdivide2VectorType<0>,
1259                 LLVMSubdivide2VectorType<0>,
1260                 llvm_i32_ty],
1261                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1262
1263  class AdvSIMD_SVE_CDOT_LANE_Intrinsic
1264    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1265                [LLVMMatchType<0>,
1266                 LLVMSubdivide4VectorType<0>,
1267                 LLVMSubdivide4VectorType<0>,
1268                 llvm_i32_ty,
1269                 llvm_i32_ty],
1270                [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1271
1272  // NOTE: There is no relationship between these intrinsics beyond an attempt
1273  // to reuse currently identical class definitions.
1274  class AdvSIMD_SVE_LOGB_Intrinsic  : AdvSIMD_SVE_CNT_Intrinsic;
1275  class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
1276  class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;
1277
1278  // This class of intrinsics are not intended to be useful within LLVM IR but
1279  // are instead here to support some of the more regid parts of the ACLE.
1280  class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN>
1281      : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
1282}
1283
1284//===----------------------------------------------------------------------===//
1285// SVE
1286
1287let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
1288
1289class AdvSIMD_SVE_Reduce_Intrinsic
1290  : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
1291              [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1292               llvm_anyvector_ty],
1293              [IntrNoMem]>;
1294
1295class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
1296  : DefaultAttrsIntrinsic<[llvm_i64_ty],
1297              [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1298               llvm_anyvector_ty],
1299              [IntrNoMem]>;
1300
1301class AdvSIMD_SVE_WHILE_Intrinsic
1302    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1303                [llvm_anyint_ty, LLVMMatchType<1>],
1304                [IntrNoMem]>;
1305
1306class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
1307    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1308                [
1309                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1310                  LLVMPointerToElt<0>,
1311                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1312                ],
1313                [IntrReadMem, IntrArgMemOnly]>;
1314
1315class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
1316    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1317                [
1318                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1319                  LLVMPointerToElt<0>,
1320                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1321                ],
1322                [IntrReadMem, IntrArgMemOnly]>;
1323
1324class AdvSIMD_GatherLoad_VS_Intrinsic
1325    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1326                [
1327                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1328                  llvm_anyvector_ty,
1329                  llvm_i64_ty
1330                ],
1331                [IntrReadMem]>;
1332
1333class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
1334    : DefaultAttrsIntrinsic<[],
1335               [
1336                 llvm_anyvector_ty,
1337                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1338                 LLVMPointerToElt<0>,
1339                 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1340               ],
1341               [IntrWriteMem, IntrArgMemOnly]>;
1342
1343class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
1344    : DefaultAttrsIntrinsic<[],
1345               [
1346                 llvm_anyvector_ty,
1347                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1348                 LLVMPointerToElt<0>,
1349                 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1350               ],
1351               [IntrWriteMem, IntrArgMemOnly]>;
1352
1353class AdvSIMD_ScatterStore_VS_Intrinsic
1354    : DefaultAttrsIntrinsic<[],
1355               [
1356                 llvm_anyvector_ty,
1357                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1358                 llvm_anyvector_ty, llvm_i64_ty
1359               ],
1360               [IntrWriteMem]>;
1361
1362
1363class SVE_gather_prf_SV
1364    : DefaultAttrsIntrinsic<[],
1365                [
1366                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1367                  llvm_ptr_ty, // Base address
1368                  llvm_anyvector_ty, // Offsets
1369                  llvm_i32_ty // Prfop
1370                ],
1371                [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;
1372
1373class SVE_gather_prf_VS
1374    : DefaultAttrsIntrinsic<[],
1375                [
1376                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1377                  llvm_anyvector_ty, // Base addresses
1378                  llvm_i64_ty, // Scalar offset
1379                  llvm_i32_ty // Prfop
1380                ],
1381                [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;
1382
1383class SVE_MatMul_Intrinsic
1384    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1385                [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
1386                [IntrNoMem]>;
1387
1388class SVE_4Vec_BF16
1389    : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
1390                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
1391                [IntrNoMem]>;
1392
1393class SVE_4Vec_BF16_Indexed
1394    : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
1395                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
1396                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1397
1398//
1399// Vector tuple creation intrinsics (ACLE)
1400//
1401
1402def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
1403def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
1404def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;
1405
1406//
1407// Vector tuple insertion/extraction intrinsics (ACLE)
1408//
1409
1410def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
1411def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;
1412
1413//
1414// Loads
1415//
1416
1417def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;
1418
1419def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1420def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1421def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1422
1423def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1424def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1425def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1426
1427def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1428def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
1429
1430//
1431// Stores
1432//
1433
1434def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
1435def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
1436def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
1437def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;
1438
1439def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1440
1441//
1442// Prefetches
1443//
1444
1445def int_aarch64_sve_prf
1446  : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
1447                  [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
1448
1449// Scalar + 32-bit scaled offset vector, zero extend, packed and
1450// unpacked.
1451def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
1452def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
1453def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
1454def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;
1455
1456// Scalar + 32-bit scaled offset vector, sign extend, packed and
1457// unpacked.
1458def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
1459def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
1460def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
1461def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;
1462
1463// Scalar + 64-bit scaled offset vector.
1464def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
1465def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
1466def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
1467def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;
1468
1469// Vector + scalar.
1470def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
1471def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
1472def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
1473def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;
1474
1475//
1476// Scalar to vector operations
1477//
1478
1479def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
1480def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;
1481
1482
1483def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;
1484
1485//
1486// Address calculation
1487//
1488
1489def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
1490def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
1491def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
1492def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;
1493
1494//
1495// Integer arithmetic
1496//
1497
1498def int_aarch64_sve_add   : AdvSIMD_Pred2VectorArg_Intrinsic;
1499def int_aarch64_sve_sub   : AdvSIMD_Pred2VectorArg_Intrinsic;
1500def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1501
1502def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;
1503
1504def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
1505def int_aarch64_sve_mul_lane   : AdvSIMD_2VectorArgIndexed_Intrinsic;
1506def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1507def int_aarch64_sve_umulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1508
1509def int_aarch64_sve_sdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1510def int_aarch64_sve_udiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1511def int_aarch64_sve_sdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1512def int_aarch64_sve_udivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1513
1514def int_aarch64_sve_smax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1515def int_aarch64_sve_umax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1516def int_aarch64_sve_smin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1517def int_aarch64_sve_umin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1518def int_aarch64_sve_sabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1519def int_aarch64_sve_uabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1520
1521def int_aarch64_sve_mad        : AdvSIMD_Pred3VectorArg_Intrinsic;
1522def int_aarch64_sve_msb        : AdvSIMD_Pred3VectorArg_Intrinsic;
1523def int_aarch64_sve_mla        : AdvSIMD_Pred3VectorArg_Intrinsic;
1524def int_aarch64_sve_mla_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1525def int_aarch64_sve_mls        : AdvSIMD_Pred3VectorArg_Intrinsic;
1526def int_aarch64_sve_mls_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1527
1528def int_aarch64_sve_saddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1529def int_aarch64_sve_uaddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1530
1531def int_aarch64_sve_smaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1532def int_aarch64_sve_umaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1533def int_aarch64_sve_sminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1534def int_aarch64_sve_uminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1535
1536def int_aarch64_sve_orv        : AdvSIMD_SVE_Reduce_Intrinsic;
1537def int_aarch64_sve_eorv       : AdvSIMD_SVE_Reduce_Intrinsic;
1538def int_aarch64_sve_andv       : AdvSIMD_SVE_Reduce_Intrinsic;
1539
1540def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
1541def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
1542
1543def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
1544def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1545
1546def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
1547def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1548
1549def int_aarch64_sve_sqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1550def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1551def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1552def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1553
1554// Shifts
1555
1556def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1557def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1558def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
1559def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
1560def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
1561def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1562def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1563def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1564
1565//
1566// Integer comparisons
1567//
1568
1569def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1570def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
1571def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1572def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
1573def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
1574def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
1575
1576def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1577def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1578def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1579def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1580def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1581def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1582def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1583def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1584def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1585def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1586
1587//
1588// Counting bits
1589//
1590
1591def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
1592def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
1593def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
1594
1595//
1596// Counting elements
1597//
1598
1599def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
1600def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
1601def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
1602def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
1603
1604def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
1605
1606//
1607// FFR manipulation
1608//
1609
1610def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], []>;
1611def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
1612def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  DefaultAttrsIntrinsic<[], []>;
1613def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty]>;
1614
1615//
1616// Saturating scalar arithmetic
1617//
1618
1619def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1620def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1621def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1622def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1623
1624def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1625def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1626def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1627def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1628def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1629def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1630def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1631def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1632def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1633def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1634
1635def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1636def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1637def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1638def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1639
1640def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1641def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1642def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1643def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1644def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1645def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1646def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1647def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1648def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1649def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1650
1651def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1652def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1653def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1654def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1655
1656def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1657def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1658def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1659def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1660def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1661def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1662def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1663def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1664def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1665def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1666
1667def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1668def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1669def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1670def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1671
1672def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1673def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1674def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1675def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1676def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1677def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1678def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1679def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1680def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1681def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1682
1683//
1684// Reversal
1685//
1686
1687def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
1688def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
1689def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
1690def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;
1691
1692//
1693// Permutations and selection
1694//
1695
1696def int_aarch64_sve_clasta    : AdvSIMD_Pred2VectorArg_Intrinsic;
1697def int_aarch64_sve_clasta_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1698def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
1699def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1700def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
1701def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
1702def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
1703def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
1704def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
1705def int_aarch64_sve_lastb     : AdvSIMD_SVE_Reduce_Intrinsic;
1706def int_aarch64_sve_rev       : AdvSIMD_1VectorArg_Intrinsic;
1707def int_aarch64_sve_splice    : AdvSIMD_Pred2VectorArg_Intrinsic;
1708def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1709def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1710def int_aarch64_sve_tbl       : AdvSIMD_SVE_TBL_Intrinsic;
1711def int_aarch64_sve_trn1      : AdvSIMD_2VectorArg_Intrinsic;
1712def int_aarch64_sve_trn2      : AdvSIMD_2VectorArg_Intrinsic;
1713def int_aarch64_sve_trn1q     : AdvSIMD_2VectorArg_Intrinsic;
1714def int_aarch64_sve_trn2q     : AdvSIMD_2VectorArg_Intrinsic;
1715def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1716def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1717def int_aarch64_sve_uzp1      : AdvSIMD_2VectorArg_Intrinsic;
1718def int_aarch64_sve_uzp2      : AdvSIMD_2VectorArg_Intrinsic;
1719def int_aarch64_sve_uzp1q     : AdvSIMD_2VectorArg_Intrinsic;
1720def int_aarch64_sve_uzp2q     : AdvSIMD_2VectorArg_Intrinsic;
1721def int_aarch64_sve_zip1      : AdvSIMD_2VectorArg_Intrinsic;
1722def int_aarch64_sve_zip2      : AdvSIMD_2VectorArg_Intrinsic;
1723def int_aarch64_sve_zip1q     : AdvSIMD_2VectorArg_Intrinsic;
1724def int_aarch64_sve_zip2q     : AdvSIMD_2VectorArg_Intrinsic;
1725
1726//
1727// Logical operations
1728//
1729
1730def int_aarch64_sve_and  : AdvSIMD_Pred2VectorArg_Intrinsic;
1731def int_aarch64_sve_bic  : AdvSIMD_Pred2VectorArg_Intrinsic;
1732def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
1733def int_aarch64_sve_eor  : AdvSIMD_Pred2VectorArg_Intrinsic;
1734def int_aarch64_sve_not  : AdvSIMD_Merged1VectorArg_Intrinsic;
1735def int_aarch64_sve_orr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1736
1737//
1738// Conversion
1739//
1740
1741def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1742def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1743def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1744def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1745def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1746def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1747
1748//
1749// While comparisons
1750//
1751
1752def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
1753def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
1754def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
1755def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
1756def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
1757def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
1758def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
1759def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;
1760
1761//
1762// Floating-point arithmetic
1763//
1764
1765def int_aarch64_sve_fabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1766def int_aarch64_sve_fabs       : AdvSIMD_Merged1VectorArg_Intrinsic;
1767def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1768def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
1769def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
1770def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
1771def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1772def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1773def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
1774def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
1775def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1776def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1777def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1778def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1779def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
1780def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1781def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
1782def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1783def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;
1784def int_aarch64_sve_fmul       : AdvSIMD_Pred2VectorArg_Intrinsic;
1785def int_aarch64_sve_fmulx      : AdvSIMD_Pred2VectorArg_Intrinsic;
1786def int_aarch64_sve_fneg       : AdvSIMD_Merged1VectorArg_Intrinsic;
1787def int_aarch64_sve_fmul_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
1788def int_aarch64_sve_fnmad      : AdvSIMD_Pred3VectorArg_Intrinsic;
1789def int_aarch64_sve_fnmla      : AdvSIMD_Pred3VectorArg_Intrinsic;
1790def int_aarch64_sve_fnmls      : AdvSIMD_Pred3VectorArg_Intrinsic;
1791def int_aarch64_sve_fnmsb      : AdvSIMD_Pred3VectorArg_Intrinsic;
1792def int_aarch64_sve_frecpe_x   : AdvSIMD_1VectorArg_Intrinsic;
1793def int_aarch64_sve_frecps_x   : AdvSIMD_2VectorArg_Intrinsic;
1794def int_aarch64_sve_frecpx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1795def int_aarch64_sve_frinta     : AdvSIMD_Merged1VectorArg_Intrinsic;
1796def int_aarch64_sve_frinti     : AdvSIMD_Merged1VectorArg_Intrinsic;
1797def int_aarch64_sve_frintm     : AdvSIMD_Merged1VectorArg_Intrinsic;
1798def int_aarch64_sve_frintn     : AdvSIMD_Merged1VectorArg_Intrinsic;
1799def int_aarch64_sve_frintp     : AdvSIMD_Merged1VectorArg_Intrinsic;
1800def int_aarch64_sve_frintx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1801def int_aarch64_sve_frintz     : AdvSIMD_Merged1VectorArg_Intrinsic;
1802def int_aarch64_sve_frsqrte_x  : AdvSIMD_1VectorArg_Intrinsic;
1803def int_aarch64_sve_frsqrts_x  : AdvSIMD_2VectorArg_Intrinsic;
1804def int_aarch64_sve_fscale     : AdvSIMD_SVE_SCALE_Intrinsic;
1805def int_aarch64_sve_fsqrt      : AdvSIMD_Merged1VectorArg_Intrinsic;
1806def int_aarch64_sve_fsub       : AdvSIMD_Pred2VectorArg_Intrinsic;
1807def int_aarch64_sve_fsubr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1808def int_aarch64_sve_ftmad_x    : AdvSIMD_2VectorArgIndexed_Intrinsic;
1809def int_aarch64_sve_ftsmul_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1810def int_aarch64_sve_ftssel_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1811
1812//
1813// Floating-point reductions
1814//
1815
1816def int_aarch64_sve_fadda   : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1817def int_aarch64_sve_faddv   : AdvSIMD_SVE_Reduce_Intrinsic;
1818def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
1819def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1820def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
1821def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1822
1823//
1824// Floating-point conversions
1825//
1826
1827def int_aarch64_sve_fcvt   : AdvSIMD_SVE_FCVT_Intrinsic;
1828def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
1829def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
1830def int_aarch64_sve_scvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1831def int_aarch64_sve_ucvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1832
1833//
1834// Floating-point comparisons
1835//
1836
1837def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
1838def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;
1839
1840def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1841def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
1842def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1843def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
1844def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;
1845
1846def int_aarch64_sve_fcvtzs_i32f16   : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1847def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1848def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1849def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1850
1851def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<"svcvt_bf16_f32_m",   llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1852def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1853
1854def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1855def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1856def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1857def int_aarch64_sve_fcvtzu_i64f32   : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1858
1859def int_aarch64_sve_fcvt_f16f32     : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1860def int_aarch64_sve_fcvt_f16f64     : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1861def int_aarch64_sve_fcvt_f32f64     : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1862
1863def int_aarch64_sve_fcvt_f32f16     : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1864def int_aarch64_sve_fcvt_f64f16     : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1865def int_aarch64_sve_fcvt_f64f32     : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1866
1867def int_aarch64_sve_fcvtlt_f32f16   : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1868def int_aarch64_sve_fcvtlt_f64f32   : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1869def int_aarch64_sve_fcvtnt_f16f32   : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1870def int_aarch64_sve_fcvtnt_f32f64   : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1871
1872def int_aarch64_sve_fcvtx_f32f64    : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1873def int_aarch64_sve_fcvtxnt_f32f64  : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1874
1875def int_aarch64_sve_scvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1876def int_aarch64_sve_scvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1877def int_aarch64_sve_scvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1878def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1879
1880def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1881def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1882def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1883def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1884
1885//
1886// Predicate creation
1887//
1888
1889def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
1890
1891//
1892// Predicate operations
1893//
1894
1895def int_aarch64_sve_and_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1896def int_aarch64_sve_bic_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1897def int_aarch64_sve_brka    : AdvSIMD_Merged1VectorArg_Intrinsic;
1898def int_aarch64_sve_brka_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1899def int_aarch64_sve_brkb    : AdvSIMD_Merged1VectorArg_Intrinsic;
1900def int_aarch64_sve_brkb_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1901def int_aarch64_sve_brkn_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1902def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1903def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1904def int_aarch64_sve_eor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1905def int_aarch64_sve_nand_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1906def int_aarch64_sve_nor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1907def int_aarch64_sve_orn_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1908def int_aarch64_sve_orr_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1909def int_aarch64_sve_pfirst  : AdvSIMD_Pred1VectorArg_Intrinsic;
1910def int_aarch64_sve_pnext   : AdvSIMD_Pred1VectorArg_Intrinsic;
1911def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1912def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1913
1914//
1915// Testing predicates
1916//
1917
1918def int_aarch64_sve_ptest_any   : AdvSIMD_SVE_PTEST_Intrinsic;
1919def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
1920def int_aarch64_sve_ptest_last  : AdvSIMD_SVE_PTEST_Intrinsic;
1921
1922//
1923// Reinterpreting data
1924//
1925
1926def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
1927                                                    [llvm_nxv16i1_ty],
1928                                                    [IntrNoMem]>;
1929
1930def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty],
1931                                                  [llvm_anyvector_ty],
1932                                                  [IntrNoMem]>;
1933
1934//
1935// Gather loads: scalar base + vector offsets
1936//
1937
1938// 64 bit unscaled offsets
1939def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1940
1941// 64 bit scaled offsets
1942def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1943
1944// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1945def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1946def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1947
1948// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1949def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1950def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1951
1952//
1953// Gather loads: vector base + scalar offset
1954//
1955
1956def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1957
1958
1959//
1960// First-faulting gather loads: scalar base + vector offsets
1961//
1962
1963// 64 bit unscaled offsets
1964def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1965
1966// 64 bit scaled offsets
1967def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1968
1969// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1970def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1971def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1972
1973// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1974def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1975def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1976
1977//
1978// First-faulting gather loads: vector base + scalar offset
1979//
1980
1981def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1982
1983
1984//
1985// Non-temporal gather loads: scalar base + vector offsets
1986//
1987
1988// 64 bit unscaled offsets
1989def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1990
1991// 64 bit indices
1992def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1993
1994// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
1995def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1996
1997//
1998// Non-temporal gather loads: vector base + scalar offset
1999//
2000
2001def int_aarch64_sve_ldnt1_gather_scalar_offset  : AdvSIMD_GatherLoad_VS_Intrinsic;
2002
2003//
2004// Scatter stores: scalar base + vector offsets
2005//
2006
2007// 64 bit unscaled offsets
2008def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2009
2010// 64 bit scaled offsets
2011def int_aarch64_sve_st1_scatter_index
2012    : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2013
2014// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2015def int_aarch64_sve_st1_scatter_sxtw
2016    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2017
2018def int_aarch64_sve_st1_scatter_uxtw
2019    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2020
2021// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
2022def int_aarch64_sve_st1_scatter_sxtw_index
2023    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2024
2025def int_aarch64_sve_st1_scatter_uxtw_index
2026    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2027
2028//
2029// Scatter stores: vector base + scalar offset
2030//
2031
2032def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
2033
2034//
2035// Non-temporal scatter stores: scalar base + vector offsets
2036//
2037
2038// 64 bit unscaled offsets
2039def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2040
2041// 64 bit indices
2042def int_aarch64_sve_stnt1_scatter_index
2043    : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2044
2045// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2046def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2047
2048//
2049// Non-temporal scatter stores: vector base + scalar offset
2050//
2051
2052def int_aarch64_sve_stnt1_scatter_scalar_offset  : AdvSIMD_ScatterStore_VS_Intrinsic;
2053
2054//
2055// SVE2 - Uniform DSP operations
2056//
2057
2058def int_aarch64_sve_saba          : AdvSIMD_3VectorArg_Intrinsic;
2059def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2060def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2061def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2062def int_aarch64_sve_sli           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2063def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
2064def int_aarch64_sve_sqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2065def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
2066def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
2067def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
2068def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
2069def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2070def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
2071def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2072def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
2073def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2074def int_aarch64_sve_sqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2075def int_aarch64_sve_sqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2076def int_aarch64_sve_sqshlu        : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2077def int_aarch64_sve_sqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2078def int_aarch64_sve_sqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2079def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2080def int_aarch64_sve_sri           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2081def int_aarch64_sve_srshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2082def int_aarch64_sve_srshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2083def int_aarch64_sve_srsra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2084def int_aarch64_sve_ssra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2085def int_aarch64_sve_suqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2086def int_aarch64_sve_uaba          : AdvSIMD_3VectorArg_Intrinsic;
2087def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2088def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2089def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2090def int_aarch64_sve_uqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2091def int_aarch64_sve_uqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2092def int_aarch64_sve_uqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2093def int_aarch64_sve_uqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2094def int_aarch64_sve_uqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2095def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
2096def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2097def int_aarch64_sve_urshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2098def int_aarch64_sve_urshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2099def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;
2100def int_aarch64_sve_ursra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2101def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2102def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2103
2104//
2105// SVE2 - Widening DSP operations
2106//
2107
2108def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
2109def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
2110def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
2111def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
2112def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
2113def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
2114def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
2115def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
2116def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
2117def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
2118def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
2119def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
2120def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
2121def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
2122def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
2123def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
2124def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
2125def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
2126def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
2127def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
2128def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
2129def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
2130def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
2131def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
2132def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
2133def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
2134def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
2135def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;
2136
2137//
2138// SVE2 - Non-widening pairwise arithmetic
2139//
2140
2141def int_aarch64_sve_addp    : AdvSIMD_Pred2VectorArg_Intrinsic;
2142def int_aarch64_sve_faddp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2143def int_aarch64_sve_fmaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2144def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2145def int_aarch64_sve_fminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2146def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2147def int_aarch64_sve_smaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2148def int_aarch64_sve_sminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2149def int_aarch64_sve_umaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2150def int_aarch64_sve_uminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2151
2152//
2153// SVE2 - Widening pairwise arithmetic
2154//
2155
2156def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2157def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2158
2159//
2160// SVE2 - Uniform complex integer arithmetic
2161//
2162
2163def int_aarch64_sve_cadd_x           : AdvSIMD_SVE2_CADD_Intrinsic;
2164def int_aarch64_sve_sqcadd_x         : AdvSIMD_SVE2_CADD_Intrinsic;
2165def int_aarch64_sve_cmla_x           : AdvSIMD_SVE2_CMLA_Intrinsic;
2166def int_aarch64_sve_cmla_lane_x      : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2167def int_aarch64_sve_sqrdcmlah_x      : AdvSIMD_SVE2_CMLA_Intrinsic;
2168def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2169
2170//
2171// SVE2 - Widening complex integer arithmetic
2172//
2173
2174def int_aarch64_sve_saddlbt   : SVE2_2VectorArg_Long_Intrinsic;
2175def int_aarch64_sve_ssublbt   : SVE2_2VectorArg_Long_Intrinsic;
2176def int_aarch64_sve_ssubltb   : SVE2_2VectorArg_Long_Intrinsic;
2177
2178//
2179// SVE2 - Widening complex integer dot product
2180//
2181
2182def int_aarch64_sve_cdot      : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2183def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;
2184
2185//
2186// SVE2 - Floating-point widening multiply-accumulate
2187//
2188
2189def int_aarch64_sve_fmlalb        : SVE2_3VectorArg_Long_Intrinsic;
2190def int_aarch64_sve_fmlalb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2191def int_aarch64_sve_fmlalt        : SVE2_3VectorArg_Long_Intrinsic;
2192def int_aarch64_sve_fmlalt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2193def int_aarch64_sve_fmlslb        : SVE2_3VectorArg_Long_Intrinsic;
2194def int_aarch64_sve_fmlslb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2195def int_aarch64_sve_fmlslt        : SVE2_3VectorArg_Long_Intrinsic;
2196def int_aarch64_sve_fmlslt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2197
2198//
2199// SVE2 - Floating-point integer binary logarithm
2200//
2201
2202def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;
2203
2204//
2205// SVE2 - Vector histogram count
2206//
2207
2208def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
2209def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;
2210
2211//
2212// SVE2 - Character match
2213//
2214
2215def int_aarch64_sve_match   : AdvSIMD_SVE_Compare_Intrinsic;
2216def int_aarch64_sve_nmatch  : AdvSIMD_SVE_Compare_Intrinsic;
2217
2218//
2219// SVE2 - Unary narrowing operations
2220//
2221
2222def int_aarch64_sve_sqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2223def int_aarch64_sve_sqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2224def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
2225def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2226def int_aarch64_sve_uqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2227def int_aarch64_sve_uqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2228
2229//
2230// SVE2 - Binary narrowing DSP operations
2231//
2232def int_aarch64_sve_addhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2233def int_aarch64_sve_addhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2234
2235def int_aarch64_sve_raddhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2236def int_aarch64_sve_raddhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2237
2238def int_aarch64_sve_subhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2239def int_aarch64_sve_subhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2240
2241def int_aarch64_sve_rsubhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2242def int_aarch64_sve_rsubhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2243
2244// Narrowing shift right
2245def int_aarch64_sve_shrnb     : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2246def int_aarch64_sve_shrnt     : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2247
2248def int_aarch64_sve_rshrnb    : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2249def int_aarch64_sve_rshrnt    : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2250
2251// Saturating shift right - signed input/output
2252def int_aarch64_sve_sqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2253def int_aarch64_sve_sqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2254
2255def int_aarch64_sve_sqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2256def int_aarch64_sve_sqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2257
2258// Saturating shift right - unsigned input/output
2259def int_aarch64_sve_uqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2260def int_aarch64_sve_uqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2261
2262def int_aarch64_sve_uqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2263def int_aarch64_sve_uqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2264
2265// Saturating shift right - signed input, unsigned output
2266def int_aarch64_sve_sqshrunb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2267def int_aarch64_sve_sqshrunt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2268
2269def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2270def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2271
2272// SVE2 MLA LANE.
2273def int_aarch64_sve_smlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2274def int_aarch64_sve_smlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2275def int_aarch64_sve_umlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2276def int_aarch64_sve_umlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2277def int_aarch64_sve_smlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2278def int_aarch64_sve_smlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2279def int_aarch64_sve_umlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2280def int_aarch64_sve_umlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2281def int_aarch64_sve_smullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2282def int_aarch64_sve_smullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2283def int_aarch64_sve_umullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2284def int_aarch64_sve_umullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2285def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2286def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2287def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2288def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2289def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2290def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2291
2292// SVE2 MLA Unpredicated.
2293def int_aarch64_sve_smlalb      : SVE2_3VectorArg_Long_Intrinsic;
2294def int_aarch64_sve_smlalt      : SVE2_3VectorArg_Long_Intrinsic;
2295def int_aarch64_sve_umlalb      : SVE2_3VectorArg_Long_Intrinsic;
2296def int_aarch64_sve_umlalt      : SVE2_3VectorArg_Long_Intrinsic;
2297def int_aarch64_sve_smlslb      : SVE2_3VectorArg_Long_Intrinsic;
2298def int_aarch64_sve_smlslt      : SVE2_3VectorArg_Long_Intrinsic;
2299def int_aarch64_sve_umlslb      : SVE2_3VectorArg_Long_Intrinsic;
2300def int_aarch64_sve_umlslt      : SVE2_3VectorArg_Long_Intrinsic;
2301def int_aarch64_sve_smullb      : SVE2_2VectorArg_Long_Intrinsic;
2302def int_aarch64_sve_smullt      : SVE2_2VectorArg_Long_Intrinsic;
2303def int_aarch64_sve_umullb      : SVE2_2VectorArg_Long_Intrinsic;
2304def int_aarch64_sve_umullt      : SVE2_2VectorArg_Long_Intrinsic;
2305
2306def int_aarch64_sve_sqdmlalb    : SVE2_3VectorArg_Long_Intrinsic;
2307def int_aarch64_sve_sqdmlalt    : SVE2_3VectorArg_Long_Intrinsic;
2308def int_aarch64_sve_sqdmlslb    : SVE2_3VectorArg_Long_Intrinsic;
2309def int_aarch64_sve_sqdmlslt    : SVE2_3VectorArg_Long_Intrinsic;
2310def int_aarch64_sve_sqdmullb    : SVE2_2VectorArg_Long_Intrinsic;
2311def int_aarch64_sve_sqdmullt    : SVE2_2VectorArg_Long_Intrinsic;
2312def int_aarch64_sve_sqdmlalbt   : SVE2_3VectorArg_Long_Intrinsic;
2313def int_aarch64_sve_sqdmlslbt   : SVE2_3VectorArg_Long_Intrinsic;
2314
2315// SVE2 ADDSUB Long Unpredicated.
2316def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
2317def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
2318def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
2319def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;
2320
2321//
2322// SVE2 - Polynomial arithmetic
2323//
2324def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
2325def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
2326def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
2327def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
2328
2329//
2330// SVE2 bitwise ternary operations.
2331//
2332def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
2333def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
2334def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
2335def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
2336def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
2337def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
2338def int_aarch64_sve_xar    : AdvSIMD_2VectorArgIndexed_Intrinsic;
2339
2340//
2341// SVE2 - Optional AES, SHA-3 and SM4
2342//
2343
2344def int_aarch64_sve_aesd    : GCCBuiltin<"__builtin_sve_svaesd_u8">,
2345                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2346                                        [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2347                                        [IntrNoMem]>;
2348def int_aarch64_sve_aesimc  : GCCBuiltin<"__builtin_sve_svaesimc_u8">,
2349                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2350                                        [llvm_nxv16i8_ty],
2351                                        [IntrNoMem]>;
2352def int_aarch64_sve_aese    : GCCBuiltin<"__builtin_sve_svaese_u8">,
2353                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2354                                        [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2355                                        [IntrNoMem]>;
2356def int_aarch64_sve_aesmc   : GCCBuiltin<"__builtin_sve_svaesmc_u8">,
2357                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
2358                                        [llvm_nxv16i8_ty],
2359                                        [IntrNoMem]>;
2360def int_aarch64_sve_rax1    : GCCBuiltin<"__builtin_sve_svrax1_u64">,
2361                              DefaultAttrsIntrinsic<[llvm_nxv2i64_ty],
2362                                        [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
2363                                        [IntrNoMem]>;
2364def int_aarch64_sve_sm4e    : GCCBuiltin<"__builtin_sve_svsm4e_u32">,
2365                              DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
2366                                        [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2367                                        [IntrNoMem]>;
2368def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">,
2369                              DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
2370                                        [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2371                                        [IntrNoMem]>;
2372//
2373// SVE2 - Extended table lookup/permute
2374//
2375
2376def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
2377def int_aarch64_sve_tbx  : AdvSIMD_SVE2_TBX_Intrinsic;
2378
2379//
2380// SVE2 - Optional bit permutation
2381//
2382
2383def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
2384def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
2385def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;
2386
2387
2388//
2389// SVE ACLE: 7.3. INT8 matrix multiply extensions
2390//
2391def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
2392def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
2393def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;
2394
2395def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
2396def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2397def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2398
2399//
2400// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
2401//
2402def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
2403
2404//
2405// SVE ACLE: 7.2. BFloat16 extensions
2406//
2407
2408def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
2409def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
2410def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
2411
2412def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;
2413
2414def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
2415def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
2416def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
2417}
2418
2419//
2420// SVE2 - Contiguous conflict detection
2421//
2422
2423def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2424def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2425def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2426def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2427def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2428def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2429def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2430def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;
2431