xref: /freebsd/contrib/llvm-project/llvm/include/llvm/IR/IntrinsicsAArch64.td (revision a0409676120c1e558d0ade943019934e0f15118d)
1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines all of the AARCH64-specific intrinsics.
10//
11//===----------------------------------------------------------------------===//
12
13let TargetPrefix = "aarch64" in {
14
15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
16def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
17def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
18def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
19
20def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
21def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
22def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
23                               [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
24def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
25                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
26
27def int_aarch64_clrex : Intrinsic<[]>;
28
29def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
30                                LLVMMatchType<0>], [IntrNoMem]>;
31def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
32                                LLVMMatchType<0>], [IntrNoMem]>;
33
34def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
35
36def int_aarch64_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
37def int_aarch64_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
38
39//===----------------------------------------------------------------------===//
40// HINT
41
42def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
43
44//===----------------------------------------------------------------------===//
45// Data Barrier Instructions
46
47def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
48def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
49def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
50
51// A space-consuming intrinsic primarily for testing block and jump table
52// placements. The first argument is the number of bytes this "instruction"
53// takes up, the second and return value are essentially chains, used to force
54// ordering during ISel.
55def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
56
57}
58
59//===----------------------------------------------------------------------===//
60// Advanced SIMD (NEON)
61
62let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
63  class AdvSIMD_2Scalar_Float_Intrinsic
64    : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
65                [IntrNoMem]>;
66
67  class AdvSIMD_FPToIntRounding_Intrinsic
68    : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
69
70  class AdvSIMD_1IntArg_Intrinsic
71    : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
72  class AdvSIMD_1FloatArg_Intrinsic
73    : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
74  class AdvSIMD_1VectorArg_Intrinsic
75    : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
76  class AdvSIMD_1VectorArg_Expand_Intrinsic
77    : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
78  class AdvSIMD_1VectorArg_Long_Intrinsic
79    : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
80  class AdvSIMD_1IntArg_Narrow_Intrinsic
81    : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
82  class AdvSIMD_1VectorArg_Narrow_Intrinsic
83    : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
84  class AdvSIMD_1VectorArg_Int_Across_Intrinsic
85    : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
86  class AdvSIMD_1VectorArg_Float_Across_Intrinsic
87    : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
88
89  class AdvSIMD_2IntArg_Intrinsic
90    : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
91                [IntrNoMem]>;
92  class AdvSIMD_2FloatArg_Intrinsic
93    : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
94                [IntrNoMem]>;
95  class AdvSIMD_2VectorArg_Intrinsic
96    : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
97                [IntrNoMem]>;
98  class AdvSIMD_2VectorArg_Compare_Intrinsic
99    : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
100                [IntrNoMem]>;
101  class AdvSIMD_2Arg_FloatCompare_Intrinsic
102    : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
103                [IntrNoMem]>;
104  class AdvSIMD_2VectorArg_Long_Intrinsic
105    : Intrinsic<[llvm_anyvector_ty],
106                [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
107                [IntrNoMem]>;
108  class AdvSIMD_2VectorArg_Wide_Intrinsic
109    : Intrinsic<[llvm_anyvector_ty],
110                [LLVMMatchType<0>, LLVMTruncatedType<0>],
111                [IntrNoMem]>;
112  class AdvSIMD_2VectorArg_Narrow_Intrinsic
113    : Intrinsic<[llvm_anyvector_ty],
114                [LLVMExtendedType<0>, LLVMExtendedType<0>],
115                [IntrNoMem]>;
116  class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
117    : Intrinsic<[llvm_anyint_ty],
118                [LLVMExtendedType<0>, llvm_i32_ty],
119                [IntrNoMem]>;
120  class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
121    : Intrinsic<[llvm_anyvector_ty],
122                [llvm_anyvector_ty],
123                [IntrNoMem]>;
124  class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
125    : Intrinsic<[llvm_anyvector_ty],
126                [LLVMTruncatedType<0>],
127                [IntrNoMem]>;
128  class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
129    : Intrinsic<[llvm_anyvector_ty],
130                [LLVMTruncatedType<0>, llvm_i32_ty],
131                [IntrNoMem]>;
132  class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
133    : Intrinsic<[llvm_anyvector_ty],
134                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
135                [IntrNoMem]>;
136  class AdvSIMD_2VectorArg_Lane_Intrinsic
137    : Intrinsic<[llvm_anyint_ty],
138                [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
139                [IntrNoMem]>;
140
141  class AdvSIMD_3VectorArg_Intrinsic
142      : Intrinsic<[llvm_anyvector_ty],
143               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
144               [IntrNoMem]>;
145  class AdvSIMD_3VectorArg_Scalar_Intrinsic
146      : Intrinsic<[llvm_anyvector_ty],
147               [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
148               [IntrNoMem]>;
149  class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
150      : Intrinsic<[llvm_anyvector_ty],
151               [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
152                LLVMMatchType<1>], [IntrNoMem]>;
153  class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
154    : Intrinsic<[llvm_anyvector_ty],
155                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
156                [IntrNoMem]>;
157  class AdvSIMD_CvtFxToFP_Intrinsic
158    : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
159                [IntrNoMem]>;
160  class AdvSIMD_CvtFPToFx_Intrinsic
161    : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
162                [IntrNoMem]>;
163
164  class AdvSIMD_1Arg_Intrinsic
165    : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
166
167  class AdvSIMD_Dot_Intrinsic
168    : Intrinsic<[llvm_anyvector_ty],
169                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
170                [IntrNoMem]>;
171
172  class AdvSIMD_FP16FML_Intrinsic
173    : Intrinsic<[llvm_anyvector_ty],
174                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
175                [IntrNoMem]>;
176
177  class AdvSIMD_MatMul_Intrinsic
178    : Intrinsic<[llvm_anyvector_ty],
179                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
180                [IntrNoMem]>;
181
182  class AdvSIMD_FML_Intrinsic
183    : Intrinsic<[llvm_anyvector_ty],
184                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
185                [IntrNoMem]>;
186
187}
188
189// Arithmetic ops
190
191let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
192  // Vector Add Across Lanes
193  def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
194  def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
195  def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
196
197  // Vector Long Add Across Lanes
198  def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
199  def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
200
201  // Vector Halving Add
202  def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
203  def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
204
205  // Vector Rounding Halving Add
206  def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
207  def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
208
209  // Vector Saturating Add
210  def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
211  def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
212  def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
213  def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
214
215  // Vector Add High-Half
216  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
217  // header is no longer supported.
218  def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
219
220  // Vector Rounding Add High-Half
221  def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
222
223  // Vector Saturating Doubling Multiply High
224  def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
225  def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
226  def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
227
228  // Vector Saturating Rounding Doubling Multiply High
229  def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
230  def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
231  def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;
232
233  // Vector Polynominal Multiply
234  def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
235
236  // Vector Long Multiply
237  def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
238  def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
239  def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
240
241  // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
242  // it with a v16i8.
243  def int_aarch64_neon_pmull64 :
244        Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
245
246  // Vector Extending Multiply
247  def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
248    let IntrProperties = [IntrNoMem, Commutative];
249  }
250
251  // Vector Saturating Doubling Long Multiply
252  def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
253  def int_aarch64_neon_sqdmulls_scalar
254    : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
255
256  // Vector Halving Subtract
257  def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
258  def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
259
260  // Vector Saturating Subtract
261  def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
262  def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
263
264  // Vector Subtract High-Half
265  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
266  // header is no longer supported.
267  def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
268
269  // Vector Rounding Subtract High-Half
270  def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
271
272  // Vector Compare Absolute Greater-than-or-equal
273  def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
274
275  // Vector Compare Absolute Greater-than
276  def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
277
278  // Vector Absolute Difference
279  def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
280  def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
281  def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
282
283  // Scalar Absolute Difference
284  def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
285
286  // Vector Max
287  def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
288  def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
289  def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
290  def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
291
292  // Vector Max Across Lanes
293  def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
294  def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
295  def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
296  def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
297
298  // Vector Min
299  def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
300  def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
301  def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
302  def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
303
304  // Vector Min/Max Number
305  def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
306  def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
307
308  // Vector Min Across Lanes
309  def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
310  def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
311  def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
312  def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
313
314  // Pairwise Add
315  def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
316  def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
317
318  // Long Pairwise Add
319  // FIXME: In theory, we shouldn't need intrinsics for saddlp or
320  // uaddlp, but tblgen's type inference currently can't handle the
321  // pattern fragments this ends up generating.
322  def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
323  def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
324
325  // Folding Maximum
326  def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
327  def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
328  def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
329
330  // Folding Minimum
331  def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
332  def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
333  def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
334
335  // Reciprocal Estimate/Step
336  def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
337  def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
338
339  // Reciprocal Exponent
340  def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
341
342  // Vector Saturating Shift Left
343  def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
344  def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
345
346  // Vector Rounding Shift Left
347  def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
348  def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
349
350  // Vector Saturating Rounding Shift Left
351  def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
352  def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
353
354  // Vector Signed->Unsigned Shift Left by Constant
355  def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
356
357  // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
358  def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
359
360  // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
361  def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
362
363  // Vector Narrowing Shift Right by Constant
364  def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
365  def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
366
367  // Vector Rounding Narrowing Shift Right by Constant
368  def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
369
370  // Vector Rounding Narrowing Saturating Shift Right by Constant
371  def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
372  def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
373
374  // Vector Shift Left
375  def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
376  def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
377
378  // Vector Widening Shift Left by Constant
379  def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
380  def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
381  def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
382
383  // Vector Shift Right by Constant and Insert
384  def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
385
386  // Vector Shift Left by Constant and Insert
387  def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
388
389  // Vector Saturating Narrow
390  def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
391  def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
392  def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
393  def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
394
395  // Vector Saturating Extract and Unsigned Narrow
396  def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
397  def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
398
399  // Vector Absolute Value
400  def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
401
402  // Vector Saturating Absolute Value
403  def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
404
405  // Vector Saturating Negation
406  def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
407
408  // Vector Count Leading Sign Bits
409  def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
410
411  // Vector Reciprocal Estimate
412  def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
413  def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
414
415  // Vector Square Root Estimate
416  def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
417  def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
418
419  // Vector Bitwise Reverse
420  def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
421
422  // Vector Conversions Between Half-Precision and Single-Precision.
423  def int_aarch64_neon_vcvtfp2hf
424    : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
425  def int_aarch64_neon_vcvthf2fp
426    : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
427
428  // Vector Conversions Between Floating-point and Fixed-point.
429  def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
430  def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
431  def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
432  def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
433
434  // Vector FP->Int Conversions
435  def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
436  def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
437  def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
438  def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
439  def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
440  def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
441  def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
442  def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
443  def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
444  def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
445
446  // Vector FP Rounding: only ties to even is unrepresented by a normal
447  // intrinsic.
448  def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
449
450  // Scalar FP->Int conversions
451
452  // Vector FP Inexact Narrowing
453  def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
454
455  // Scalar FP Inexact Narrowing
456  def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
457                                        [IntrNoMem]>;
458
459  // v8.2-A Dot Product
460  def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
461  def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
462
463// v8.6-A Matrix Multiply Intrinsics
464  def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
465  def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
466  def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
467  def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
468  def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
469  def int_aarch64_neon_bfmmla : AdvSIMD_MatMul_Intrinsic;
470  def int_aarch64_neon_bfmlalb : AdvSIMD_FML_Intrinsic;
471  def int_aarch64_neon_bfmlalt : AdvSIMD_FML_Intrinsic;
472
473
474  // v8.6-A Bfloat Intrinsics
475  def int_aarch64_neon_bfcvt
476    : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
477  def int_aarch64_neon_bfcvtn
478    : Intrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
479  def int_aarch64_neon_bfcvtn2
480    : Intrinsic<[llvm_v8bf16_ty],
481                [llvm_v8bf16_ty, llvm_v4f32_ty],
482                [IntrNoMem]>;
483
484  // v8.2-A FP16 Fused Multiply-Add Long
485  def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
486  def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
487  def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
488  def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
489
490  // v8.3-A Floating-point complex add
491  def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
492  def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;
493}
494
495let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
496  class AdvSIMD_2Vector2Index_Intrinsic
497    : Intrinsic<[llvm_anyvector_ty],
498                [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
499                [IntrNoMem]>;
500}
501
502// Vector element to element moves
503def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
504
505let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
506  class AdvSIMD_1Vec_Load_Intrinsic
507      : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
508                  [IntrReadMem, IntrArgMemOnly]>;
509  class AdvSIMD_1Vec_Store_Lane_Intrinsic
510    : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
511                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
512
513  class AdvSIMD_2Vec_Load_Intrinsic
514    : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
515                [LLVMAnyPointerType<LLVMMatchType<0>>],
516                [IntrReadMem, IntrArgMemOnly]>;
517  class AdvSIMD_2Vec_Load_Lane_Intrinsic
518    : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
519                [LLVMMatchType<0>, llvm_anyvector_ty,
520                 llvm_i64_ty, llvm_anyptr_ty],
521                [IntrReadMem, IntrArgMemOnly]>;
522  class AdvSIMD_2Vec_Store_Intrinsic
523    : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
524                     LLVMAnyPointerType<LLVMMatchType<0>>],
525                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
526  class AdvSIMD_2Vec_Store_Lane_Intrinsic
527    : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
528                 llvm_i64_ty, llvm_anyptr_ty],
529                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
530
531  class AdvSIMD_3Vec_Load_Intrinsic
532    : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
533                [LLVMAnyPointerType<LLVMMatchType<0>>],
534                [IntrReadMem, IntrArgMemOnly]>;
535  class AdvSIMD_3Vec_Load_Lane_Intrinsic
536    : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
537                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
538                 llvm_i64_ty, llvm_anyptr_ty],
539                [IntrReadMem, IntrArgMemOnly]>;
540  class AdvSIMD_3Vec_Store_Intrinsic
541    : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
542                     LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
543                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
544  class AdvSIMD_3Vec_Store_Lane_Intrinsic
545    : Intrinsic<[], [llvm_anyvector_ty,
546                 LLVMMatchType<0>, LLVMMatchType<0>,
547                 llvm_i64_ty, llvm_anyptr_ty],
548                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
549
550  class AdvSIMD_4Vec_Load_Intrinsic
551    : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
552                 LLVMMatchType<0>, llvm_anyvector_ty],
553                [LLVMAnyPointerType<LLVMMatchType<0>>],
554                [IntrReadMem, IntrArgMemOnly]>;
555  class AdvSIMD_4Vec_Load_Lane_Intrinsic
556    : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
557                 LLVMMatchType<0>, LLVMMatchType<0>],
558                [LLVMMatchType<0>, LLVMMatchType<0>,
559                 LLVMMatchType<0>, llvm_anyvector_ty,
560                 llvm_i64_ty, llvm_anyptr_ty],
561                [IntrReadMem, IntrArgMemOnly]>;
562  class AdvSIMD_4Vec_Store_Intrinsic
563    : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
564                 LLVMMatchType<0>, LLVMMatchType<0>,
565                 LLVMAnyPointerType<LLVMMatchType<0>>],
566                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
567  class AdvSIMD_4Vec_Store_Lane_Intrinsic
568    : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
569                 LLVMMatchType<0>, LLVMMatchType<0>,
570                 llvm_i64_ty, llvm_anyptr_ty],
571                [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
572}
573
574// Memory ops
575
576def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
577def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
578def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
579
580def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
581def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
582def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
583
584def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
585def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
586def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
587
588def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
589def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
590def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
591
592def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
593def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
594def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
595
596def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
597def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
598def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
599
600def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
601def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
602def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
603
604let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
605  class AdvSIMD_Tbl1_Intrinsic
606    : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
607                [IntrNoMem]>;
608  class AdvSIMD_Tbl2_Intrinsic
609    : Intrinsic<[llvm_anyvector_ty],
610                [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
611  class AdvSIMD_Tbl3_Intrinsic
612    : Intrinsic<[llvm_anyvector_ty],
613                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
614                 LLVMMatchType<0>],
615                [IntrNoMem]>;
616  class AdvSIMD_Tbl4_Intrinsic
617    : Intrinsic<[llvm_anyvector_ty],
618                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
619                 LLVMMatchType<0>],
620                [IntrNoMem]>;
621
622  class AdvSIMD_Tbx1_Intrinsic
623    : Intrinsic<[llvm_anyvector_ty],
624                [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
625                [IntrNoMem]>;
626  class AdvSIMD_Tbx2_Intrinsic
627    : Intrinsic<[llvm_anyvector_ty],
628                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
629                 LLVMMatchType<0>],
630                [IntrNoMem]>;
631  class AdvSIMD_Tbx3_Intrinsic
632    : Intrinsic<[llvm_anyvector_ty],
633                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
634                 llvm_v16i8_ty, LLVMMatchType<0>],
635                [IntrNoMem]>;
636  class AdvSIMD_Tbx4_Intrinsic
637    : Intrinsic<[llvm_anyvector_ty],
638                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
639                 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
640                [IntrNoMem]>;
641}
642def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
643def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
644def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
645def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
646
647def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
648def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
649def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
650def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
651
652let TargetPrefix = "aarch64" in {
653  class FPCR_Get_Intrinsic
654    : Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
655}
656
657// FPCR
658def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
659
660let TargetPrefix = "aarch64" in {
661  class Crypto_AES_DataKey_Intrinsic
662    : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
663
664  class Crypto_AES_Data_Intrinsic
665    : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
666
667  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
668  // (v4i32).
669  class Crypto_SHA_5Hash4Schedule_Intrinsic
670    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
671                [IntrNoMem]>;
672
673  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
674  // (v4i32).
675  class Crypto_SHA_1Hash_Intrinsic
676    : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
677
678  // SHA intrinsic taking 8 words of the schedule
679  class Crypto_SHA_8Schedule_Intrinsic
680    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
681
682  // SHA intrinsic taking 12 words of the schedule
683  class Crypto_SHA_12Schedule_Intrinsic
684    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
685                [IntrNoMem]>;
686
687  // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
688  class Crypto_SHA_8Hash4Schedule_Intrinsic
689    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
690                [IntrNoMem]>;
691}
692
693// AES
694def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
695def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
696def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
697def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
698
699// SHA1
700def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
701def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
702def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
703def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
704
705def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
706def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
707
708// SHA256
709def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
710def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
711def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
712def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
713
714//===----------------------------------------------------------------------===//
715// CRC32
716
717let TargetPrefix = "aarch64" in {
718
719def int_aarch64_crc32b  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
720    [IntrNoMem]>;
721def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
722    [IntrNoMem]>;
723def int_aarch64_crc32h  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
724    [IntrNoMem]>;
725def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
726    [IntrNoMem]>;
727def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
728    [IntrNoMem]>;
729def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
730    [IntrNoMem]>;
731def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
732    [IntrNoMem]>;
733def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
734    [IntrNoMem]>;
735}
736
737//===----------------------------------------------------------------------===//
738// Memory Tagging Extensions (MTE) Intrinsics
739let TargetPrefix = "aarch64" in {
740def int_aarch64_irg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
741    [IntrNoMem, IntrHasSideEffects]>;
742def int_aarch64_addg  : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
743    [IntrNoMem]>;
744def int_aarch64_gmi   : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
745    [IntrNoMem]>;
746def int_aarch64_ldg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
747    [IntrReadMem]>;
748def int_aarch64_stg   : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
749    [IntrWriteMem]>;
750def int_aarch64_subp :  Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
751    [IntrNoMem]>;
752
753// The following are codegen-only intrinsics for stack instrumentation.
754
755// Generate a randomly tagged stack base pointer.
756def int_aarch64_irg_sp   : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty],
757    [IntrNoMem, IntrHasSideEffects]>;
758
759// Transfer pointer tag with offset.
760// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
761// * address is the address in ptr0
762// * tag is a function of (tag in baseptr, tag_offset).
763// Address bits in baseptr and tag bits in ptr0 are ignored.
764// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
765//   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
766// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
767def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
768    [IntrNoMem, ImmArg<ArgIndex<2>>]>;
769
770// Update allocation tags for the memory range to match the tag in the pointer argument.
771def int_aarch64_settag  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
772    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
773
774// Update allocation tags for the memory range to match the tag in the pointer argument,
775// and set memory contents to zero.
776def int_aarch64_settag_zero  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
777    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
778
779// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
780def int_aarch64_stgp  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
781    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
782}
783
784// Transactional Memory Extension (TME) Intrinsics
785let TargetPrefix = "aarch64" in {
786def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
787                         Intrinsic<[llvm_i64_ty]>;
788
789def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>;
790
791def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
792                          Intrinsic<[], [llvm_i64_ty], [ImmArg<ArgIndex<0>>]>;
793
794def int_aarch64_ttest   : GCCBuiltin<"__builtin_arm_ttest">,
795                          Intrinsic<[llvm_i64_ty], [],
796                                    [IntrNoMem, IntrHasSideEffects]>;
797}
798
799def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
800def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
801def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
802def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
803def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
804def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
805def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
806def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
807def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
808def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
809def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
810
811let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
812
813  class AdvSIMD_SVE_Create_2Vector_Tuple
814    : Intrinsic<[llvm_anyvector_ty],
815                [llvm_anyvector_ty, LLVMMatchType<1>],
816                [IntrReadMem]>;
817
818  class AdvSIMD_SVE_Create_3Vector_Tuple
819    : Intrinsic<[llvm_anyvector_ty],
820                [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
821                [IntrReadMem]>;
822
823  class AdvSIMD_SVE_Create_4Vector_Tuple
824    : Intrinsic<[llvm_anyvector_ty],
825                [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
826                 LLVMMatchType<1>],
827                [IntrReadMem]>;
828
829  class AdvSIMD_SVE_Set_Vector_Tuple
830    : Intrinsic<[llvm_anyvector_ty],
831                [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
832                [IntrReadMem, ImmArg<ArgIndex<1>>]>;
833
834  class AdvSIMD_SVE_Get_Vector_Tuple
835    : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
836                [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;
837
838  class AdvSIMD_ManyVec_PredLoad_Intrinsic
839    : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
840                [IntrReadMem, IntrArgMemOnly]>;
841
842  class AdvSIMD_1Vec_PredLoad_Intrinsic
843    : Intrinsic<[llvm_anyvector_ty],
844                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
845                 LLVMPointerToElt<0>],
846                [IntrReadMem, IntrArgMemOnly]>;
847
848  class AdvSIMD_1Vec_PredStore_Intrinsic
849    : Intrinsic<[],
850                [llvm_anyvector_ty,
851                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
852                 LLVMPointerToElt<0>],
853                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
854
855  class AdvSIMD_2Vec_PredStore_Intrinsic
856      : Intrinsic<[],
857                  [llvm_anyvector_ty, LLVMMatchType<0>,
858                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
859                  [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
860
861  class AdvSIMD_3Vec_PredStore_Intrinsic
862      : Intrinsic<[],
863                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
864                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
865                  [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
866
867  class AdvSIMD_4Vec_PredStore_Intrinsic
868      : Intrinsic<[],
869                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
870                   LLVMMatchType<0>,
871                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
872                  [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
873
874  class AdvSIMD_SVE_Index_Intrinsic
875    : Intrinsic<[llvm_anyvector_ty],
876                [LLVMVectorElementType<0>,
877                 LLVMVectorElementType<0>],
878                [IntrNoMem]>;
879
880  class AdvSIMD_Merged1VectorArg_Intrinsic
881    : Intrinsic<[llvm_anyvector_ty],
882                [LLVMMatchType<0>,
883                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
884                 LLVMMatchType<0>],
885                [IntrNoMem]>;
886
887  class AdvSIMD_2VectorArgIndexed_Intrinsic
888    : Intrinsic<[llvm_anyvector_ty],
889                [LLVMMatchType<0>,
890                 LLVMMatchType<0>,
891                 llvm_i32_ty],
892                [IntrNoMem, ImmArg<ArgIndex<2>>]>;
893
894  class AdvSIMD_3VectorArgIndexed_Intrinsic
895    : Intrinsic<[llvm_anyvector_ty],
896                [LLVMMatchType<0>,
897                 LLVMMatchType<0>,
898                 LLVMMatchType<0>,
899                 llvm_i32_ty],
900                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
901
902  class AdvSIMD_Pred1VectorArg_Intrinsic
903    : Intrinsic<[llvm_anyvector_ty],
904                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
905                 LLVMMatchType<0>],
906                [IntrNoMem]>;
907
908  class AdvSIMD_Pred2VectorArg_Intrinsic
909    : Intrinsic<[llvm_anyvector_ty],
910                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
911                 LLVMMatchType<0>,
912                 LLVMMatchType<0>],
913                [IntrNoMem]>;
914
915  class AdvSIMD_Pred3VectorArg_Intrinsic
916    : Intrinsic<[llvm_anyvector_ty],
917                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
918                 LLVMMatchType<0>,
919                 LLVMMatchType<0>,
920                 LLVMMatchType<0>],
921                [IntrNoMem]>;
922
923  class AdvSIMD_SVE_Compare_Intrinsic
924    : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
925                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
926                 llvm_anyvector_ty,
927                 LLVMMatchType<0>],
928                [IntrNoMem]>;
929
930  class AdvSIMD_SVE_CompareWide_Intrinsic
931    : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
932                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
933                 llvm_anyvector_ty,
934                 llvm_nxv2i64_ty],
935                [IntrNoMem]>;
936
937  class AdvSIMD_SVE_Saturating_Intrinsic
938    : Intrinsic<[llvm_anyvector_ty],
939                [LLVMMatchType<0>,
940                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
941                [IntrNoMem]>;
942
943  class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
944    : Intrinsic<[llvm_anyvector_ty],
945                [LLVMMatchType<0>,
946                 llvm_i32_ty,
947                 llvm_i32_ty],
948                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
949
950  class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
951    : Intrinsic<[T],
952                [T, llvm_anyvector_ty],
953                [IntrNoMem]>;
954
955  class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
956    : Intrinsic<[T],
957                [T, llvm_i32_ty, llvm_i32_ty],
958                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;
959
960  class AdvSIMD_SVE_CNT_Intrinsic
961    : Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
962                [LLVMVectorOfBitcastsToInt<0>,
963                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
964                 llvm_anyvector_ty],
965                [IntrNoMem]>;
966
967  class AdvSIMD_SVE_ReduceWithInit_Intrinsic
968    : Intrinsic<[LLVMVectorElementType<0>],
969                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
970                 LLVMVectorElementType<0>,
971                 llvm_anyvector_ty],
972                [IntrNoMem]>;
973
974  class AdvSIMD_SVE_ShiftByImm_Intrinsic
975    : Intrinsic<[llvm_anyvector_ty],
976                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
977                 LLVMMatchType<0>,
978                 llvm_i32_ty],
979                [IntrNoMem, ImmArg<ArgIndex<2>>]>;
980
981  class AdvSIMD_SVE_ShiftWide_Intrinsic
982    : Intrinsic<[llvm_anyvector_ty],
983                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
984                 LLVMMatchType<0>,
985                 llvm_nxv2i64_ty],
986                [IntrNoMem]>;
987
988  class AdvSIMD_SVE_Unpack_Intrinsic
989    : Intrinsic<[llvm_anyvector_ty],
990               [LLVMSubdivide2VectorType<0>],
991               [IntrNoMem]>;
992
993  class AdvSIMD_SVE_CADD_Intrinsic
994    : Intrinsic<[llvm_anyvector_ty],
995                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
996                 LLVMMatchType<0>,
997                 LLVMMatchType<0>,
998                 llvm_i32_ty],
999                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1000
1001  class AdvSIMD_SVE_CMLA_Intrinsic
1002    : Intrinsic<[llvm_anyvector_ty],
1003                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1004                 LLVMMatchType<0>,
1005                 LLVMMatchType<0>,
1006                 LLVMMatchType<0>,
1007                 llvm_i32_ty],
1008                [IntrNoMem, ImmArg<ArgIndex<4>>]>;
1009
1010  class AdvSIMD_SVE_CMLA_LANE_Intrinsic
1011    : Intrinsic<[llvm_anyvector_ty],
1012                [LLVMMatchType<0>,
1013                 LLVMMatchType<0>,
1014                 LLVMMatchType<0>,
1015                 llvm_i32_ty,
1016                 llvm_i32_ty],
1017                [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1018
1019  class AdvSIMD_SVE_DUP_Intrinsic
1020    : Intrinsic<[llvm_anyvector_ty],
1021                [LLVMMatchType<0>,
1022                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1023                 LLVMVectorElementType<0>],
1024                [IntrNoMem]>;
1025
1026  class AdvSIMD_SVE_DUP_Unpred_Intrinsic
1027    : Intrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
1028                [IntrNoMem]>;
1029
1030  class AdvSIMD_SVE_DUPQ_Intrinsic
1031    : Intrinsic<[llvm_anyvector_ty],
1032                [LLVMMatchType<0>,
1033                 llvm_i64_ty],
1034                [IntrNoMem]>;
1035
1036  class AdvSIMD_SVE_EXPA_Intrinsic
1037    : Intrinsic<[llvm_anyvector_ty],
1038                [LLVMVectorOfBitcastsToInt<0>],
1039                [IntrNoMem]>;
1040
1041  class AdvSIMD_SVE_FCVT_Intrinsic
1042    : Intrinsic<[llvm_anyvector_ty],
1043                [LLVMMatchType<0>,
1044                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1045                 llvm_anyvector_ty],
1046                [IntrNoMem]>;
1047
1048  class AdvSIMD_SVE_FCVTZS_Intrinsic
1049    : Intrinsic<[llvm_anyvector_ty],
1050                [LLVMVectorOfBitcastsToInt<0>,
1051                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1052                 llvm_anyvector_ty],
1053                [IntrNoMem]>;
1054
1055  class AdvSIMD_SVE_INSR_Intrinsic
1056    : Intrinsic<[llvm_anyvector_ty],
1057                [LLVMMatchType<0>,
1058                 LLVMVectorElementType<0>],
1059                [IntrNoMem]>;
1060
1061  class AdvSIMD_SVE_PTRUE_Intrinsic
1062    : Intrinsic<[llvm_anyvector_ty],
1063                [llvm_i32_ty],
1064                [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1065
1066  class AdvSIMD_SVE_PUNPKHI_Intrinsic
1067    : Intrinsic<[LLVMHalfElementsVectorType<0>],
1068                [llvm_anyvector_ty],
1069                [IntrNoMem]>;
1070
1071  class AdvSIMD_SVE_SCALE_Intrinsic
1072    : Intrinsic<[llvm_anyvector_ty],
1073                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1074                 LLVMMatchType<0>,
1075                 LLVMVectorOfBitcastsToInt<0>],
1076                [IntrNoMem]>;
1077
1078  class AdvSIMD_SVE_SCVTF_Intrinsic
1079    : Intrinsic<[llvm_anyvector_ty],
1080                [LLVMMatchType<0>,
1081                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1082                 llvm_anyvector_ty],
1083                [IntrNoMem]>;
1084
1085  class AdvSIMD_SVE_TSMUL_Intrinsic
1086    : Intrinsic<[llvm_anyvector_ty],
1087                [LLVMMatchType<0>,
1088                 LLVMVectorOfBitcastsToInt<0>],
1089                [IntrNoMem]>;
1090
1091  class AdvSIMD_SVE_CNTB_Intrinsic
1092    : Intrinsic<[llvm_i64_ty],
1093                [llvm_i32_ty],
1094                [IntrNoMem, ImmArg<ArgIndex<0>>]>;
1095
1096  class AdvSIMD_SVE_CNTP_Intrinsic
1097    : Intrinsic<[llvm_i64_ty],
1098                [llvm_anyvector_ty, LLVMMatchType<0>],
1099                [IntrNoMem]>;
1100
1101  class AdvSIMD_SVE_DOT_Intrinsic
1102    : Intrinsic<[llvm_anyvector_ty],
1103                [LLVMMatchType<0>,
1104                 LLVMSubdivide4VectorType<0>,
1105                 LLVMSubdivide4VectorType<0>],
1106                [IntrNoMem]>;
1107
1108  class AdvSIMD_SVE_DOT_Indexed_Intrinsic
1109    : Intrinsic<[llvm_anyvector_ty],
1110                [LLVMMatchType<0>,
1111                 LLVMSubdivide4VectorType<0>,
1112                 LLVMSubdivide4VectorType<0>,
1113                 llvm_i32_ty],
1114                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1115
1116  class AdvSIMD_SVE_PTEST_Intrinsic
1117    : Intrinsic<[llvm_i1_ty],
1118                [llvm_anyvector_ty,
1119                 LLVMMatchType<0>],
1120                [IntrNoMem]>;
1121
1122  class AdvSIMD_SVE_TBL_Intrinsic
1123    : Intrinsic<[llvm_anyvector_ty],
1124                [LLVMMatchType<0>,
1125                 LLVMVectorOfBitcastsToInt<0>],
1126                [IntrNoMem]>;
1127
1128  class AdvSIMD_SVE2_TBX_Intrinsic
1129    : Intrinsic<[llvm_anyvector_ty],
1130                [LLVMMatchType<0>,
1131                 LLVMMatchType<0>,
1132                 LLVMVectorOfBitcastsToInt<0>],
1133                [IntrNoMem]>;
1134
1135  class SVE2_1VectorArg_Long_Intrinsic
1136    : Intrinsic<[llvm_anyvector_ty],
1137                [LLVMSubdivide2VectorType<0>,
1138                 llvm_i32_ty],
1139                [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1140
1141  class SVE2_2VectorArg_Long_Intrinsic
1142    : Intrinsic<[llvm_anyvector_ty],
1143                [LLVMSubdivide2VectorType<0>,
1144                 LLVMSubdivide2VectorType<0>],
1145                [IntrNoMem]>;
1146
1147  class SVE2_2VectorArgIndexed_Long_Intrinsic
1148  : Intrinsic<[llvm_anyvector_ty],
1149              [LLVMSubdivide2VectorType<0>,
1150               LLVMSubdivide2VectorType<0>,
1151               llvm_i32_ty],
1152              [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1153
1154  class SVE2_2VectorArg_Wide_Intrinsic
1155    : Intrinsic<[llvm_anyvector_ty],
1156                [LLVMMatchType<0>,
1157                 LLVMSubdivide2VectorType<0>],
1158                [IntrNoMem]>;
1159
1160  class SVE2_2VectorArg_Pred_Long_Intrinsic
1161    : Intrinsic<[llvm_anyvector_ty],
1162                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1163                 LLVMMatchType<0>,
1164                 LLVMSubdivide2VectorType<0>],
1165                [IntrNoMem]>;
1166
1167  class SVE2_3VectorArg_Long_Intrinsic
1168    : Intrinsic<[llvm_anyvector_ty],
1169                [LLVMMatchType<0>,
1170                 LLVMSubdivide2VectorType<0>,
1171                 LLVMSubdivide2VectorType<0>],
1172                [IntrNoMem]>;
1173
1174  class SVE2_3VectorArgIndexed_Long_Intrinsic
1175    : Intrinsic<[llvm_anyvector_ty],
1176                [LLVMMatchType<0>,
1177                 LLVMSubdivide2VectorType<0>,
1178                 LLVMSubdivide2VectorType<0>,
1179                 llvm_i32_ty],
1180                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1181
1182  class SVE2_1VectorArg_Narrowing_Intrinsic
1183    : Intrinsic<[LLVMSubdivide2VectorType<0>],
1184                [llvm_anyvector_ty],
1185                [IntrNoMem]>;
1186
1187  class SVE2_Merged1VectorArg_Narrowing_Intrinsic
1188    : Intrinsic<[LLVMSubdivide2VectorType<0>],
1189                [LLVMSubdivide2VectorType<0>,
1190                 llvm_anyvector_ty],
1191                [IntrNoMem]>;
1192  class SVE2_2VectorArg_Narrowing_Intrinsic
1193      : Intrinsic<
1194            [LLVMSubdivide2VectorType<0>],
1195            [llvm_anyvector_ty, LLVMMatchType<0>],
1196            [IntrNoMem]>;
1197
1198  class SVE2_Merged2VectorArg_Narrowing_Intrinsic
1199      : Intrinsic<
1200            [LLVMSubdivide2VectorType<0>],
1201            [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
1202            [IntrNoMem]>;
1203
1204  class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
1205      : Intrinsic<[LLVMSubdivide2VectorType<0>],
1206                  [llvm_anyvector_ty, llvm_i32_ty],
1207                  [IntrNoMem, ImmArg<ArgIndex<1>>]>;
1208
1209  class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
1210      : Intrinsic<[LLVMSubdivide2VectorType<0>],
1211                  [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
1212                   llvm_i32_ty],
1213                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;
1214
1215  class SVE2_CONFLICT_DETECT_Intrinsic
1216    : Intrinsic<[llvm_anyvector_ty],
1217                [LLVMAnyPointerType<llvm_any_ty>,
1218                 LLVMMatchType<1>]>;
1219
1220  class SVE2_3VectorArg_Indexed_Intrinsic
1221    : Intrinsic<[llvm_anyvector_ty],
1222                [LLVMMatchType<0>,
1223                 LLVMSubdivide2VectorType<0>,
1224                 LLVMSubdivide2VectorType<0>,
1225                 llvm_i32_ty],
1226                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1227
1228  class AdvSIMD_SVE_CDOT_LANE_Intrinsic
1229    : Intrinsic<[llvm_anyvector_ty],
1230                [LLVMMatchType<0>,
1231                 LLVMSubdivide4VectorType<0>,
1232                 LLVMSubdivide4VectorType<0>,
1233                 llvm_i32_ty,
1234                 llvm_i32_ty],
1235                [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;
1236
1237  // NOTE: There is no relationship between these intrinsics beyond an attempt
1238  // to reuse currently identical class definitions.
1239  class AdvSIMD_SVE_LOGB_Intrinsic  : AdvSIMD_SVE_CNT_Intrinsic;
1240  class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
1241  class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;
1242
1243  // This class of intrinsics are not intended to be useful within LLVM IR but
1244  // are instead here to support some of the more regid parts of the ACLE.
1245  class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN>
1246      : Intrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
1247}
1248
1249//===----------------------------------------------------------------------===//
1250// SVE
1251
1252let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
1253
1254class AdvSIMD_SVE_Reduce_Intrinsic
1255  : Intrinsic<[LLVMVectorElementType<0>],
1256              [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1257               llvm_anyvector_ty],
1258              [IntrNoMem]>;
1259
1260class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
1261  : Intrinsic<[llvm_i64_ty],
1262              [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1263               llvm_anyvector_ty],
1264              [IntrNoMem]>;
1265
1266class AdvSIMD_SVE_WHILE_Intrinsic
1267    : Intrinsic<[llvm_anyvector_ty],
1268                [llvm_anyint_ty, LLVMMatchType<1>],
1269                [IntrNoMem]>;
1270
1271class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
1272    : Intrinsic<[llvm_anyvector_ty],
1273                [
1274                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1275                  LLVMPointerToElt<0>,
1276                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1277                ],
1278                [IntrReadMem, IntrArgMemOnly]>;
1279
1280class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
1281    : Intrinsic<[llvm_anyvector_ty],
1282                [
1283                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1284                  LLVMPointerToElt<0>,
1285                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1286                ],
1287                [IntrReadMem, IntrArgMemOnly]>;
1288
1289class AdvSIMD_GatherLoad_VS_Intrinsic
1290    : Intrinsic<[llvm_anyvector_ty],
1291                [
1292                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1293                  llvm_anyvector_ty,
1294                  llvm_i64_ty
1295                ],
1296                [IntrReadMem, IntrArgMemOnly]>;
1297
1298class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
1299    : Intrinsic<[],
1300               [
1301                 llvm_anyvector_ty,
1302                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1303                 LLVMPointerToElt<0>,
1304                 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
1305               ],
1306               [IntrWriteMem, IntrArgMemOnly]>;
1307
1308class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
1309    : Intrinsic<[],
1310               [
1311                 llvm_anyvector_ty,
1312                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1313                 LLVMPointerToElt<0>,
1314                 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
1315               ],
1316               [IntrWriteMem, IntrArgMemOnly]>;
1317
1318class AdvSIMD_ScatterStore_VS_Intrinsic
1319    : Intrinsic<[],
1320               [
1321                 llvm_anyvector_ty,
1322                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
1323                 llvm_anyvector_ty, llvm_i64_ty
1324               ],
1325               [IntrWriteMem, IntrArgMemOnly]>;
1326
1327
1328class SVE_gather_prf_SV
1329    : Intrinsic<[],
1330                [
1331                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1332                  llvm_ptr_ty, // Base address
1333                  llvm_anyvector_ty, // Offsets
1334                  llvm_i32_ty // Prfop
1335                ],
1336                [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;
1337
1338class SVE_gather_prf_VS
1339    : Intrinsic<[],
1340                [
1341                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
1342                  llvm_anyvector_ty, // Base addresses
1343                  llvm_i64_ty, // Scalar offset
1344                  llvm_i32_ty // Prfop
1345                ],
1346                [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;
1347
1348class SVE_MatMul_Intrinsic
1349    : Intrinsic<[llvm_anyvector_ty],
1350                [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
1351                [IntrNoMem]>;
1352
1353class SVE_4Vec_BF16
1354    : Intrinsic<[llvm_nxv4f32_ty],
1355                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
1356                [IntrNoMem]>;
1357
1358class SVE_4Vec_BF16_Indexed
1359    : Intrinsic<[llvm_nxv4f32_ty],
1360                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
1361                [IntrNoMem, ImmArg<ArgIndex<3>>]>;
1362
1363//
1364// Vector tuple creation intrinsics (ACLE)
1365//
1366
1367def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
1368def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
1369def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;
1370
1371//
1372// Vector tuple insertion/extraction intrinsics (ACLE)
1373//
1374
1375def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
1376def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;
1377
1378//
1379// Loads
1380//
1381
1382def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;
1383
1384def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1385def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1386def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
1387
1388def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1389def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1390def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
1391
1392def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
1393def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;
1394
1395//
1396// Stores
1397//
1398
1399def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
1400def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
1401def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
1402def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;
1403
1404def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;
1405
1406//
1407// Prefetches
1408//
1409
1410def int_aarch64_sve_prf
1411  : Intrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
1412                  [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;
1413
1414// Scalar + 32-bit scaled offset vector, zero extend, packed and
1415// unpacked.
1416def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
1417def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
1418def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
1419def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;
1420
1421// Scalar + 32-bit scaled offset vector, sign extend, packed and
1422// unpacked.
1423def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
1424def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
1425def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
1426def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;
1427
1428// Scalar + 64-bit scaled offset vector.
1429def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
1430def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
1431def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
1432def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;
1433
1434// Vector + scalar.
1435def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
1436def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
1437def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
1438def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;
1439
1440//
1441// Scalar to vector operations
1442//
1443
1444def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
1445def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;
1446
1447
1448def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;
1449
1450//
1451// Address calculation
1452//
1453
1454def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
1455def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
1456def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
1457def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;
1458
1459//
1460// Integer arithmetic
1461//
1462
1463def int_aarch64_sve_add   : AdvSIMD_Pred2VectorArg_Intrinsic;
1464def int_aarch64_sve_sub   : AdvSIMD_Pred2VectorArg_Intrinsic;
1465def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1466
1467def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;
1468
1469def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
1470def int_aarch64_sve_mul_lane   : AdvSIMD_2VectorArgIndexed_Intrinsic;
1471def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1472def int_aarch64_sve_umulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
1473
1474def int_aarch64_sve_sdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1475def int_aarch64_sve_udiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1476def int_aarch64_sve_sdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1477def int_aarch64_sve_udivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1478
1479def int_aarch64_sve_smax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1480def int_aarch64_sve_umax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1481def int_aarch64_sve_smin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1482def int_aarch64_sve_umin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1483def int_aarch64_sve_sabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1484def int_aarch64_sve_uabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1485
1486def int_aarch64_sve_mad        : AdvSIMD_Pred3VectorArg_Intrinsic;
1487def int_aarch64_sve_msb        : AdvSIMD_Pred3VectorArg_Intrinsic;
1488def int_aarch64_sve_mla        : AdvSIMD_Pred3VectorArg_Intrinsic;
1489def int_aarch64_sve_mla_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1490def int_aarch64_sve_mls        : AdvSIMD_Pred3VectorArg_Intrinsic;
1491def int_aarch64_sve_mls_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
1492
1493def int_aarch64_sve_saddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1494def int_aarch64_sve_uaddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
1495
1496def int_aarch64_sve_smaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1497def int_aarch64_sve_umaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
1498def int_aarch64_sve_sminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1499def int_aarch64_sve_uminv      : AdvSIMD_SVE_Reduce_Intrinsic;
1500
1501def int_aarch64_sve_orv        : AdvSIMD_SVE_Reduce_Intrinsic;
1502def int_aarch64_sve_eorv       : AdvSIMD_SVE_Reduce_Intrinsic;
1503def int_aarch64_sve_andv       : AdvSIMD_SVE_Reduce_Intrinsic;
1504
1505def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
1506def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
1507
1508def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
1509def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1510
1511def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
1512def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
1513
1514def int_aarch64_sve_sqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1515def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1516def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
1517def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
1518
1519// Shifts
1520
1521def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1522def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1523def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
1524def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
1525def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
1526def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1527def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1528def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
1529
1530//
1531// Integer comparisons
1532//
1533
1534def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1535def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
1536def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1537def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
1538def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
1539def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;
1540
1541def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1542def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1543def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1544def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1545def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1546def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1547def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1548def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1549def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1550def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
1551
1552//
1553// Counting bits
1554//
1555
1556def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
1557def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
1558def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
1559
1560//
1561// Counting elements
1562//
1563
1564def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
1565def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
1566def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
1567def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;
1568
1569def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;
1570
1571//
1572// FFR manipulation
1573//
1574
1575def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   Intrinsic<[llvm_nxv16i1_ty], []>;
1576def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>;
1577def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  Intrinsic<[], []>;
1578def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   Intrinsic<[], [llvm_nxv16i1_ty]>;
1579
1580//
1581// Saturating scalar arithmetic
1582//
1583
1584def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1585def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1586def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1587def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1588
1589def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1590def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1591def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1592def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1593def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1594def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1595def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1596def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1597def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1598def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1599
1600def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1601def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1602def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1603def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1604
1605def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1606def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1607def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1608def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1609def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1610def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1611def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1612def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1613def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1614def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1615
1616def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1617def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1618def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1619def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;
1620
1621def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1622def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1623def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1624def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1625def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1626def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1627def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1628def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1629def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1630def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1631
1632def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1633def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1634def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
1635def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;
1636
1637def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1638def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1639def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1640def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1641def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1642def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1643def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
1644def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
1645def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
1646def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;
1647
1648//
1649// Reversal
1650//
1651
1652def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
1653def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
1654def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
1655def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;
1656
1657//
1658// Permutations and selection
1659//
1660
1661def int_aarch64_sve_clasta    : AdvSIMD_Pred2VectorArg_Intrinsic;
1662def int_aarch64_sve_clasta_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1663def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
1664def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1665def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
1666def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
1667def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
1668def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
1669def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
1670def int_aarch64_sve_lastb     : AdvSIMD_SVE_Reduce_Intrinsic;
1671def int_aarch64_sve_rev       : AdvSIMD_1VectorArg_Intrinsic;
1672def int_aarch64_sve_splice    : AdvSIMD_Pred2VectorArg_Intrinsic;
1673def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1674def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1675def int_aarch64_sve_tbl       : AdvSIMD_SVE_TBL_Intrinsic;
1676def int_aarch64_sve_trn1      : AdvSIMD_2VectorArg_Intrinsic;
1677def int_aarch64_sve_trn2      : AdvSIMD_2VectorArg_Intrinsic;
1678def int_aarch64_sve_trn1q     : AdvSIMD_2VectorArg_Intrinsic;
1679def int_aarch64_sve_trn2q     : AdvSIMD_2VectorArg_Intrinsic;
1680def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
1681def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
1682def int_aarch64_sve_uzp1      : AdvSIMD_2VectorArg_Intrinsic;
1683def int_aarch64_sve_uzp2      : AdvSIMD_2VectorArg_Intrinsic;
1684def int_aarch64_sve_uzp1q     : AdvSIMD_2VectorArg_Intrinsic;
1685def int_aarch64_sve_uzp2q     : AdvSIMD_2VectorArg_Intrinsic;
1686def int_aarch64_sve_zip1      : AdvSIMD_2VectorArg_Intrinsic;
1687def int_aarch64_sve_zip2      : AdvSIMD_2VectorArg_Intrinsic;
1688def int_aarch64_sve_zip1q     : AdvSIMD_2VectorArg_Intrinsic;
1689def int_aarch64_sve_zip2q     : AdvSIMD_2VectorArg_Intrinsic;
1690
1691//
1692// Logical operations
1693//
1694
1695def int_aarch64_sve_and  : AdvSIMD_Pred2VectorArg_Intrinsic;
1696def int_aarch64_sve_bic  : AdvSIMD_Pred2VectorArg_Intrinsic;
1697def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
1698def int_aarch64_sve_eor  : AdvSIMD_Pred2VectorArg_Intrinsic;
1699def int_aarch64_sve_not  : AdvSIMD_Merged1VectorArg_Intrinsic;
1700def int_aarch64_sve_orr  : AdvSIMD_Pred2VectorArg_Intrinsic;
1701
1702//
1703// Conversion
1704//
1705
1706def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1707def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1708def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1709def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
1710def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
1711def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
1712
1713//
1714// While comparisons
1715//
1716
1717def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
1718def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
1719def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
1720def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
1721def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
1722def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
1723def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
1724def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;
1725
1726//
1727// Floating-point arithmetic
1728//
1729
1730def int_aarch64_sve_fabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1731def int_aarch64_sve_fabs       : AdvSIMD_Merged1VectorArg_Intrinsic;
1732def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
1733def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
1734def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
1735def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
1736def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
1737def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1738def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
1739def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
1740def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
1741def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1742def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
1743def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
1744def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
1745def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1746def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
1747def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
1748def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;
1749def int_aarch64_sve_fmul       : AdvSIMD_Pred2VectorArg_Intrinsic;
1750def int_aarch64_sve_fmulx      : AdvSIMD_Pred2VectorArg_Intrinsic;
1751def int_aarch64_sve_fneg       : AdvSIMD_Merged1VectorArg_Intrinsic;
1752def int_aarch64_sve_fmul_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
1753def int_aarch64_sve_fnmad      : AdvSIMD_Pred3VectorArg_Intrinsic;
1754def int_aarch64_sve_fnmla      : AdvSIMD_Pred3VectorArg_Intrinsic;
1755def int_aarch64_sve_fnmls      : AdvSIMD_Pred3VectorArg_Intrinsic;
1756def int_aarch64_sve_fnmsb      : AdvSIMD_Pred3VectorArg_Intrinsic;
1757def int_aarch64_sve_frecpe_x   : AdvSIMD_1VectorArg_Intrinsic;
1758def int_aarch64_sve_frecps_x   : AdvSIMD_2VectorArg_Intrinsic;
1759def int_aarch64_sve_frecpx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1760def int_aarch64_sve_frinta     : AdvSIMD_Merged1VectorArg_Intrinsic;
1761def int_aarch64_sve_frinti     : AdvSIMD_Merged1VectorArg_Intrinsic;
1762def int_aarch64_sve_frintm     : AdvSIMD_Merged1VectorArg_Intrinsic;
1763def int_aarch64_sve_frintn     : AdvSIMD_Merged1VectorArg_Intrinsic;
1764def int_aarch64_sve_frintp     : AdvSIMD_Merged1VectorArg_Intrinsic;
1765def int_aarch64_sve_frintx     : AdvSIMD_Merged1VectorArg_Intrinsic;
1766def int_aarch64_sve_frintz     : AdvSIMD_Merged1VectorArg_Intrinsic;
1767def int_aarch64_sve_frsqrte_x  : AdvSIMD_1VectorArg_Intrinsic;
1768def int_aarch64_sve_frsqrts_x  : AdvSIMD_2VectorArg_Intrinsic;
1769def int_aarch64_sve_fscale     : AdvSIMD_SVE_SCALE_Intrinsic;
1770def int_aarch64_sve_fsqrt      : AdvSIMD_Merged1VectorArg_Intrinsic;
1771def int_aarch64_sve_fsub       : AdvSIMD_Pred2VectorArg_Intrinsic;
1772def int_aarch64_sve_fsubr      : AdvSIMD_Pred2VectorArg_Intrinsic;
1773def int_aarch64_sve_ftmad_x    : AdvSIMD_2VectorArgIndexed_Intrinsic;
1774def int_aarch64_sve_ftsmul_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1775def int_aarch64_sve_ftssel_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
1776
1777//
1778// Floating-point reductions
1779//
1780
1781def int_aarch64_sve_fadda   : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
1782def int_aarch64_sve_faddv   : AdvSIMD_SVE_Reduce_Intrinsic;
1783def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
1784def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1785def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
1786def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
1787
1788//
1789// Floating-point conversions
1790//
1791
1792def int_aarch64_sve_fcvt   : AdvSIMD_SVE_FCVT_Intrinsic;
1793def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
1794def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
1795def int_aarch64_sve_scvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1796def int_aarch64_sve_ucvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
1797
1798//
1799// Floating-point comparisons
1800//
1801
1802def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
1803def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;
1804
1805def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
1806def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
1807def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
1808def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
1809def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;
1810
1811def int_aarch64_sve_fcvtzs_i32f16   : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1812def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1813def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1814def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1815
1816def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<"svcvt_bf16_f32_m",   llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1817def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
1818
1819def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1820def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1821def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1822def int_aarch64_sve_fcvtzu_i64f32   : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1823
1824def int_aarch64_sve_fcvt_f16f32     : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1825def int_aarch64_sve_fcvt_f16f64     : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1826def int_aarch64_sve_fcvt_f32f64     : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1827
1828def int_aarch64_sve_fcvt_f32f16     : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1829def int_aarch64_sve_fcvt_f64f16     : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
1830def int_aarch64_sve_fcvt_f64f32     : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1831
1832def int_aarch64_sve_fcvtlt_f32f16   : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
1833def int_aarch64_sve_fcvtlt_f64f32   : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
1834def int_aarch64_sve_fcvtnt_f16f32   : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
1835def int_aarch64_sve_fcvtnt_f32f64   : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1836
1837def int_aarch64_sve_fcvtx_f32f64    : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1838def int_aarch64_sve_fcvtxnt_f32f64  : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
1839
1840def int_aarch64_sve_scvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1841def int_aarch64_sve_scvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1842def int_aarch64_sve_scvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1843def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1844
1845def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
1846def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1847def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
1848def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;
1849
1850//
1851// Predicate creation
1852//
1853
1854def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;
1855
1856//
1857// Predicate operations
1858//
1859
1860def int_aarch64_sve_and_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1861def int_aarch64_sve_bic_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1862def int_aarch64_sve_brka    : AdvSIMD_Merged1VectorArg_Intrinsic;
1863def int_aarch64_sve_brka_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1864def int_aarch64_sve_brkb    : AdvSIMD_Merged1VectorArg_Intrinsic;
1865def int_aarch64_sve_brkb_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
1866def int_aarch64_sve_brkn_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1867def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1868def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
1869def int_aarch64_sve_eor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1870def int_aarch64_sve_nand_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
1871def int_aarch64_sve_nor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1872def int_aarch64_sve_orn_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1873def int_aarch64_sve_orr_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
1874def int_aarch64_sve_pfirst  : AdvSIMD_Pred1VectorArg_Intrinsic;
1875def int_aarch64_sve_pnext   : AdvSIMD_Pred1VectorArg_Intrinsic;
1876def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1877def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
1878
1879//
1880// Testing predicates
1881//
1882
1883def int_aarch64_sve_ptest_any   : AdvSIMD_SVE_PTEST_Intrinsic;
1884def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
1885def int_aarch64_sve_ptest_last  : AdvSIMD_SVE_PTEST_Intrinsic;
1886
1887//
1888// Reinterpreting data
1889//
1890
1891def int_aarch64_sve_convert_from_svbool : Intrinsic<[llvm_anyvector_ty],
1892                                                    [llvm_nxv16i1_ty],
1893                                                    [IntrNoMem]>;
1894
1895def int_aarch64_sve_convert_to_svbool : Intrinsic<[llvm_nxv16i1_ty],
1896                                                  [llvm_anyvector_ty],
1897                                                  [IntrNoMem]>;
1898
1899//
1900// Gather loads: scalar base + vector offsets
1901//
1902
1903// 64 bit unscaled offsets
1904def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1905
1906// 64 bit scaled offsets
1907def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1908
1909// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1910def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1911def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1912
1913// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1914def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1915def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1916
1917//
1918// Gather loads: vector base + scalar offset
1919//
1920
1921def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1922
1923
1924//
1925// First-faulting gather loads: scalar base + vector offsets
1926//
1927
1928// 64 bit unscaled offsets
1929def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1930
1931// 64 bit scaled offsets
1932def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1933
1934// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1935def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1936def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1937
1938// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
1939def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1940def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1941
1942//
1943// First-faulting gather loads: vector base + scalar offset
1944//
1945
1946def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;
1947
1948
1949//
1950// Non-temporal gather loads: scalar base + vector offsets
1951//
1952
1953// 64 bit unscaled offsets
1954def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1955
1956// 64 bit indices
1957def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;
1958
1959// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
1960def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
1961
1962//
1963// Non-temporal gather loads: vector base + scalar offset
1964//
1965
1966def int_aarch64_sve_ldnt1_gather_scalar_offset  : AdvSIMD_GatherLoad_VS_Intrinsic;
1967
1968//
1969// Scatter stores: scalar base + vector offsets
1970//
1971
1972// 64 bit unscaled offsets
1973def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
1974
1975// 64 bit scaled offsets
1976def int_aarch64_sve_st1_scatter_index
1977    : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
1978
1979// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1980def int_aarch64_sve_st1_scatter_sxtw
1981    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1982
1983def int_aarch64_sve_st1_scatter_uxtw
1984    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1985
1986// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
1987def int_aarch64_sve_st1_scatter_sxtw_index
1988    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1989
1990def int_aarch64_sve_st1_scatter_uxtw_index
1991    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
1992
1993//
1994// Scatter stores: vector base + scalar offset
1995//
1996
1997def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;
1998
1999//
2000// Non-temporal scatter stores: scalar base + vector offsets
2001//
2002
2003// 64 bit unscaled offsets
2004def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2005
2006// 64 bit indices
2007def int_aarch64_sve_stnt1_scatter_index
2008    : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;
2009
2010// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
2011def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;
2012
2013//
2014// Non-temporal scatter stores: vector base + scalar offset
2015//
2016
2017def int_aarch64_sve_stnt1_scatter_scalar_offset  : AdvSIMD_ScatterStore_VS_Intrinsic;
2018
2019//
2020// SVE2 - Uniform DSP operations
2021//
2022
2023def int_aarch64_sve_saba          : AdvSIMD_3VectorArg_Intrinsic;
2024def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2025def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2026def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2027def int_aarch64_sve_sli           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2028def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
2029def int_aarch64_sve_sqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2030def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
2031def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
2032def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
2033def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
2034def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2035def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
2036def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
2037def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
2038def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
2039def int_aarch64_sve_sqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2040def int_aarch64_sve_sqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2041def int_aarch64_sve_sqshlu        : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2042def int_aarch64_sve_sqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2043def int_aarch64_sve_sqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2044def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2045def int_aarch64_sve_sri           : AdvSIMD_2VectorArgIndexed_Intrinsic;
2046def int_aarch64_sve_srshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2047def int_aarch64_sve_srshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2048def int_aarch64_sve_srsra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2049def int_aarch64_sve_ssra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2050def int_aarch64_sve_suqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2051def int_aarch64_sve_uaba          : AdvSIMD_3VectorArg_Intrinsic;
2052def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2053def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2054def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2055def int_aarch64_sve_uqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
2056def int_aarch64_sve_uqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
2057def int_aarch64_sve_uqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2058def int_aarch64_sve_uqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
2059def int_aarch64_sve_uqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
2060def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
2061def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2062def int_aarch64_sve_urshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
2063def int_aarch64_sve_urshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
2064def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;
2065def int_aarch64_sve_ursra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
2066def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
2067def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
2068
2069//
2070// SVE2 - Widening DSP operations
2071//
2072
2073def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
2074def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
2075def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
2076def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
2077def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
2078def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
2079def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
2080def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
2081def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
2082def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
2083def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
2084def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
2085def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
2086def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
2087def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
2088def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
2089def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
2090def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
2091def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
2092def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
2093def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
2094def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
2095def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
2096def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
2097def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
2098def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
2099def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
2100def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;
2101
2102//
2103// SVE2 - Non-widening pairwise arithmetic
2104//
2105
2106def int_aarch64_sve_addp    : AdvSIMD_Pred2VectorArg_Intrinsic;
2107def int_aarch64_sve_faddp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2108def int_aarch64_sve_fmaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2109def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2110def int_aarch64_sve_fminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2111def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
2112def int_aarch64_sve_smaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2113def int_aarch64_sve_sminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2114def int_aarch64_sve_umaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2115def int_aarch64_sve_uminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
2116
2117//
2118// SVE2 - Widening pairwise arithmetic
2119//
2120
2121def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2122def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
2123
2124//
2125// SVE2 - Uniform complex integer arithmetic
2126//
2127
2128def int_aarch64_sve_cadd_x           : AdvSIMD_SVE2_CADD_Intrinsic;
2129def int_aarch64_sve_sqcadd_x         : AdvSIMD_SVE2_CADD_Intrinsic;
2130def int_aarch64_sve_cmla_x           : AdvSIMD_SVE2_CMLA_Intrinsic;
2131def int_aarch64_sve_cmla_lane_x      : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2132def int_aarch64_sve_sqrdcmlah_x      : AdvSIMD_SVE2_CMLA_Intrinsic;
2133def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
2134
2135//
2136// SVE2 - Widening complex integer arithmetic
2137//
2138
2139def int_aarch64_sve_saddlbt   : SVE2_2VectorArg_Long_Intrinsic;
2140def int_aarch64_sve_ssublbt   : SVE2_2VectorArg_Long_Intrinsic;
2141def int_aarch64_sve_ssubltb   : SVE2_2VectorArg_Long_Intrinsic;
2142
2143//
2144// SVE2 - Widening complex integer dot product
2145//
2146
2147def int_aarch64_sve_cdot      : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2148def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;
2149
2150//
2151// SVE2 - Floating-point widening multiply-accumulate
2152//
2153
2154def int_aarch64_sve_fmlalb        : SVE2_3VectorArg_Long_Intrinsic;
2155def int_aarch64_sve_fmlalb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2156def int_aarch64_sve_fmlalt        : SVE2_3VectorArg_Long_Intrinsic;
2157def int_aarch64_sve_fmlalt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2158def int_aarch64_sve_fmlslb        : SVE2_3VectorArg_Long_Intrinsic;
2159def int_aarch64_sve_fmlslb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2160def int_aarch64_sve_fmlslt        : SVE2_3VectorArg_Long_Intrinsic;
2161def int_aarch64_sve_fmlslt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
2162
2163//
2164// SVE2 - Floating-point integer binary logarithm
2165//
2166
2167def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;
2168
2169//
2170// SVE2 - Vector histogram count
2171//
2172
2173def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
2174def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;
2175
2176//
2177// SVE2 - Character match
2178//
2179
2180def int_aarch64_sve_match   : AdvSIMD_SVE_Compare_Intrinsic;
2181def int_aarch64_sve_nmatch  : AdvSIMD_SVE_Compare_Intrinsic;
2182
2183//
2184// SVE2 - Unary narrowing operations
2185//
2186
2187def int_aarch64_sve_sqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2188def int_aarch64_sve_sqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2189def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
2190def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2191def int_aarch64_sve_uqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
2192def int_aarch64_sve_uqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
2193
2194//
2195// SVE2 - Binary narrowing DSP operations
2196//
2197def int_aarch64_sve_addhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2198def int_aarch64_sve_addhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2199
2200def int_aarch64_sve_raddhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2201def int_aarch64_sve_raddhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2202
2203def int_aarch64_sve_subhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
2204def int_aarch64_sve_subhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2205
2206def int_aarch64_sve_rsubhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
2207def int_aarch64_sve_rsubhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;
2208
2209// Narrowing shift right
2210def int_aarch64_sve_shrnb     : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2211def int_aarch64_sve_shrnt     : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2212
2213def int_aarch64_sve_rshrnb    : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2214def int_aarch64_sve_rshrnt    : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2215
2216// Saturating shift right - signed input/output
2217def int_aarch64_sve_sqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2218def int_aarch64_sve_sqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2219
2220def int_aarch64_sve_sqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2221def int_aarch64_sve_sqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2222
2223// Saturating shift right - unsigned input/output
2224def int_aarch64_sve_uqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2225def int_aarch64_sve_uqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2226
2227def int_aarch64_sve_uqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2228def int_aarch64_sve_uqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2229
2230// Saturating shift right - signed input, unsigned output
2231def int_aarch64_sve_sqshrunb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2232def int_aarch64_sve_sqshrunt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2233
2234def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
2235def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;
2236
2237// SVE2 MLA LANE.
2238def int_aarch64_sve_smlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2239def int_aarch64_sve_smlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2240def int_aarch64_sve_umlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2241def int_aarch64_sve_umlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2242def int_aarch64_sve_smlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2243def int_aarch64_sve_smlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2244def int_aarch64_sve_umlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2245def int_aarch64_sve_umlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
2246def int_aarch64_sve_smullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2247def int_aarch64_sve_smullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2248def int_aarch64_sve_umullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2249def int_aarch64_sve_umullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
2250def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2251def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2252def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2253def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
2254def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2255def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
2256
2257// SVE2 MLA Unpredicated.
2258def int_aarch64_sve_smlalb      : SVE2_3VectorArg_Long_Intrinsic;
2259def int_aarch64_sve_smlalt      : SVE2_3VectorArg_Long_Intrinsic;
2260def int_aarch64_sve_umlalb      : SVE2_3VectorArg_Long_Intrinsic;
2261def int_aarch64_sve_umlalt      : SVE2_3VectorArg_Long_Intrinsic;
2262def int_aarch64_sve_smlslb      : SVE2_3VectorArg_Long_Intrinsic;
2263def int_aarch64_sve_smlslt      : SVE2_3VectorArg_Long_Intrinsic;
2264def int_aarch64_sve_umlslb      : SVE2_3VectorArg_Long_Intrinsic;
2265def int_aarch64_sve_umlslt      : SVE2_3VectorArg_Long_Intrinsic;
2266def int_aarch64_sve_smullb      : SVE2_2VectorArg_Long_Intrinsic;
2267def int_aarch64_sve_smullt      : SVE2_2VectorArg_Long_Intrinsic;
2268def int_aarch64_sve_umullb      : SVE2_2VectorArg_Long_Intrinsic;
2269def int_aarch64_sve_umullt      : SVE2_2VectorArg_Long_Intrinsic;
2270
2271def int_aarch64_sve_sqdmlalb    : SVE2_3VectorArg_Long_Intrinsic;
2272def int_aarch64_sve_sqdmlalt    : SVE2_3VectorArg_Long_Intrinsic;
2273def int_aarch64_sve_sqdmlslb    : SVE2_3VectorArg_Long_Intrinsic;
2274def int_aarch64_sve_sqdmlslt    : SVE2_3VectorArg_Long_Intrinsic;
2275def int_aarch64_sve_sqdmullb    : SVE2_2VectorArg_Long_Intrinsic;
2276def int_aarch64_sve_sqdmullt    : SVE2_2VectorArg_Long_Intrinsic;
2277def int_aarch64_sve_sqdmlalbt   : SVE2_3VectorArg_Long_Intrinsic;
2278def int_aarch64_sve_sqdmlslbt   : SVE2_3VectorArg_Long_Intrinsic;
2279
2280// SVE2 ADDSUB Long Unpredicated.
2281def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
2282def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
2283def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
2284def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;
2285
2286//
2287// SVE2 - Polynomial arithmetic
2288//
2289def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
2290def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
2291def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
2292def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;
2293
2294//
2295// SVE2 bitwise ternary operations.
2296//
2297def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
2298def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
2299def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
2300def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
2301def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
2302def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
2303def int_aarch64_sve_xar    : AdvSIMD_2VectorArgIndexed_Intrinsic;
2304
2305//
2306// SVE2 - Optional AES, SHA-3 and SM4
2307//
2308
2309def int_aarch64_sve_aesd    : GCCBuiltin<"__builtin_sve_svaesd_u8">,
2310                              Intrinsic<[llvm_nxv16i8_ty],
2311                                        [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2312                                        [IntrNoMem]>;
2313def int_aarch64_sve_aesimc  : GCCBuiltin<"__builtin_sve_svaesimc_u8">,
2314                              Intrinsic<[llvm_nxv16i8_ty],
2315                                        [llvm_nxv16i8_ty],
2316                                        [IntrNoMem]>;
2317def int_aarch64_sve_aese    : GCCBuiltin<"__builtin_sve_svaese_u8">,
2318                              Intrinsic<[llvm_nxv16i8_ty],
2319                                        [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
2320                                        [IntrNoMem]>;
2321def int_aarch64_sve_aesmc   : GCCBuiltin<"__builtin_sve_svaesmc_u8">,
2322                              Intrinsic<[llvm_nxv16i8_ty],
2323                                        [llvm_nxv16i8_ty],
2324                                        [IntrNoMem]>;
2325def int_aarch64_sve_rax1    : GCCBuiltin<"__builtin_sve_svrax1_u64">,
2326                              Intrinsic<[llvm_nxv2i64_ty],
2327                                        [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
2328                                        [IntrNoMem]>;
2329def int_aarch64_sve_sm4e    : GCCBuiltin<"__builtin_sve_svsm4e_u32">,
2330                              Intrinsic<[llvm_nxv4i32_ty],
2331                                        [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2332                                        [IntrNoMem]>;
2333def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">,
2334                              Intrinsic<[llvm_nxv4i32_ty],
2335                                        [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
2336                                        [IntrNoMem]>;
2337//
2338// SVE2 - Extended table lookup/permute
2339//
2340
2341def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
2342def int_aarch64_sve_tbx  : AdvSIMD_SVE2_TBX_Intrinsic;
2343
2344//
2345// SVE2 - Optional bit permutation
2346//
2347
2348def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
2349def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
2350def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;
2351
2352
2353//
2354// SVE ACLE: 7.3. INT8 matrix multiply extensions
2355//
2356def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
2357def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
2358def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;
2359
2360def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
2361def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2362def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
2363
2364//
2365// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
2366//
2367def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;
2368
2369//
2370// SVE ACLE: 7.2. BFloat16 extensions
2371//
2372
2373def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
2374def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
2375def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;
2376
2377def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;
2378
2379def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
2380def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
2381def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
2382}
2383
2384//
2385// SVE2 - Contiguous conflict detection
2386//
2387
2388def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
2389def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
2390def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
2391def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
2392def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
2393def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
2394def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
2395def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;
2396