xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AArch64/SVEInstrFormats.td (revision c1d255d3ffdbe447de3ab875bf4e7d7accc5bfc5)
1//=-- SVEInstrFormats.td -  AArch64 SVE Instruction classes -*- tablegen -*--=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions.
10//
11//===----------------------------------------------------------------------===//
12
13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [
14  SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>,
15  SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>,
16  SDTCisVT<4, OtherVT>
17]>;
18
19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>;
20
21def SVEPatternOperand : AsmOperandClass {
22  let Name = "SVEPattern";
23  let ParserMethod = "tryParseSVEPattern";
24  let PredicateMethod = "isSVEPattern";
25  let RenderMethod = "addImmOperands";
26  let DiagnosticType = "InvalidSVEPattern";
27}
28
29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{
30  return (((uint32_t)Imm) < 32);
31  }]> {
32
33  let PrintMethod = "printSVEPattern";
34  let ParserMatchClass = SVEPatternOperand;
35}
36
37def SVEPrefetchOperand : AsmOperandClass {
38  let Name = "SVEPrefetch";
39  let ParserMethod = "tryParsePrefetch<true>";
40  let PredicateMethod = "isPrefetch";
41  let RenderMethod = "addPrefetchOperands";
42}
43
44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{
45    return (((uint32_t)Imm) <= 15);
46  }]> {
47  let PrintMethod = "printPrefetchOp<true>";
48  let ParserMatchClass = SVEPrefetchOperand;
49}
50
51class SVELogicalImmOperand<int Width> : AsmOperandClass {
52  let Name = "SVELogicalImm" # Width;
53  let DiagnosticType = "LogicalSecondSource";
54  let PredicateMethod = "isLogicalImm<int" # Width # "_t>";
55  let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>";
56}
57
58def sve_logical_imm8 : Operand<i64> {
59  let ParserMatchClass = SVELogicalImmOperand<8>;
60  let PrintMethod = "printLogicalImm<int8_t>";
61
62  let MCOperandPredicate = [{
63    if (!MCOp.isImm())
64      return false;
65    int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
66    return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val);
67  }];
68}
69
70def sve_logical_imm16 : Operand<i64> {
71  let ParserMatchClass = SVELogicalImmOperand<16>;
72  let PrintMethod = "printLogicalImm<int16_t>";
73
74  let MCOperandPredicate = [{
75    if (!MCOp.isImm())
76      return false;
77    int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
78    return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val);
79  }];
80}
81
82def sve_logical_imm32 : Operand<i64> {
83  let ParserMatchClass = SVELogicalImmOperand<32>;
84  let PrintMethod = "printLogicalImm<int32_t>";
85
86  let MCOperandPredicate = [{
87    if (!MCOp.isImm())
88      return false;
89    int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
90    return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val);
91  }];
92}
93
94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass {
95  let Name = "SVEPreferredLogicalImm" # Width;
96  let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>";
97  let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>";
98}
99
100def sve_preferred_logical_imm16 : Operand<i64> {
101  let ParserMatchClass = SVEPreferredLogicalImmOperand<16>;
102  let PrintMethod = "printSVELogicalImm<int16_t>";
103
104  let MCOperandPredicate = [{
105    if (!MCOp.isImm())
106      return false;
107    int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
108    return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) &&
109           AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);
110  }];
111}
112
113def sve_preferred_logical_imm32 : Operand<i64> {
114  let ParserMatchClass =  SVEPreferredLogicalImmOperand<32>;
115  let PrintMethod = "printSVELogicalImm<int32_t>";
116
117  let MCOperandPredicate = [{
118    if (!MCOp.isImm())
119      return false;
120    int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
121    return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) &&
122           AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);
123  }];
124}
125
126def sve_preferred_logical_imm64 : Operand<i64> {
127  let ParserMatchClass = SVEPreferredLogicalImmOperand<64>;
128  let PrintMethod = "printSVELogicalImm<int64_t>";
129
130  let MCOperandPredicate = [{
131    if (!MCOp.isImm())
132      return false;
133    int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64);
134    return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) &&
135           AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val);
136  }];
137}
138
139class SVELogicalImmNotOperand<int Width> : AsmOperandClass {
140  let Name = "SVELogicalImm" # Width # "Not";
141  let DiagnosticType = "LogicalSecondSource";
142  let PredicateMethod = "isLogicalImm<int" # Width # "_t>";
143  let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>";
144}
145
146def sve_logical_imm8_not : Operand<i64> {
147  let ParserMatchClass = SVELogicalImmNotOperand<8>;
148}
149
150def sve_logical_imm16_not : Operand<i64> {
151  let ParserMatchClass = SVELogicalImmNotOperand<16>;
152}
153
154def sve_logical_imm32_not : Operand<i64> {
155  let ParserMatchClass = SVELogicalImmNotOperand<32>;
156}
157
158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate>
159    : AsmOperandClass {
160  let Name = "SVE" # Infix # "Imm" # ElementWidth;
161  let DiagnosticType = "Invalid" # Name;
162  let RenderMethod = "addImmWithOptionalShiftOperands<8>";
163  let ParserMethod = "tryParseImmWithOptionalShift";
164  let PredicateMethod = Predicate;
165}
166
167def SVECpyImmOperand8  : SVEShiftedImmOperand<8,  "Cpy", "isSVECpyImm<int8_t>">;
168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">;
169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">;
170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">;
171
172def SVEAddSubImmOperand8  : SVEShiftedImmOperand<8,  "AddSub", "isSVEAddSubImm<int8_t>">;
173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">;
174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">;
175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">;
176
177class imm8_opt_lsl<int ElementWidth, string printType,
178                   AsmOperandClass OpndClass>
179    : Operand<i32> {
180  let EncoderMethod = "getImm8OptLsl";
181  let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">";
182  let PrintMethod = "printImm8OptLsl<" # printType # ">";
183  let ParserMatchClass = OpndClass;
184  let MIOperandInfo = (ops i32imm, i32imm);
185}
186
187def cpy_imm8_opt_lsl_i8  : imm8_opt_lsl<8,  "int8_t",  SVECpyImmOperand8>;
188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>;
189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>;
190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>;
191
192def addsub_imm8_opt_lsl_i8  : imm8_opt_lsl<8,  "uint8_t",  SVEAddSubImmOperand8>;
193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>;
194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>;
195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>;
196
197def SVEAddSubImm8Pat  : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>;
198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>;
199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>;
200def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>;
201
202def SVELogicalImm8Pat  : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>;
203def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>;
204def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>;
205def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>;
206
207def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>;
208
209def SVEArithUImm8Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>;
210def SVEArithUImm16Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>;
211def SVEArithUImm32Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>;
212def SVEArithUImm64Pat  : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>;
213def SVEArithSImmPat  : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>;
214
215def SVEShiftImmL8  : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>",  []>;
216def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>;
217def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>;
218def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>;
219def SVEShiftImmR8  : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8,  true>", []>;
220def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>;
221def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>;
222def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>;
223
224class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass {
225  let Name = "SVEExactFPImmOperand" # Suffix;
226  let DiagnosticType = "Invalid" # Name;
227  let ParserMethod = "tryParseFPImm<false>";
228  let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">";
229  let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">";
230}
231
232class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> {
233  let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">";
234  let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>;
235}
236
237def sve_fpimm_half_one
238    : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half",
239                           "AArch64ExactFPImm::one">;
240def sve_fpimm_half_two
241    : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half",
242                           "AArch64ExactFPImm::two">;
243def sve_fpimm_zero_one
244    : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero",
245                           "AArch64ExactFPImm::one">;
246
247def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{
248  return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
249}]> {
250  let ParserMatchClass = Imm1_16Operand;
251  let EncoderMethod = "getSVEIncDecImm";
252  let DecoderMethod = "DecodeSVEIncDecImm";
253}
254
255// This allows i32 immediate extraction from i64 based arithmetic.
256def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">;
257def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">;
258
259//===----------------------------------------------------------------------===//
260// SVE PTrue - These are used extensively throughout the pattern matching so
261//             it's important we define them first.
262//===----------------------------------------------------------------------===//
263
264class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,
265                    ValueType vt, SDPatternOperator op>
266: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern),
267  asm, "\t$Pd, $pattern",
268  "",
269  [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> {
270  bits<4> Pd;
271  bits<5> pattern;
272  let Inst{31-24} = 0b00100101;
273  let Inst{23-22} = sz8_64;
274  let Inst{21-19} = 0b011;
275  let Inst{18-17} = opc{2-1};
276  let Inst{16}    = opc{0};
277  let Inst{15-10} = 0b111000;
278  let Inst{9-5}   = pattern;
279  let Inst{4}     = 0b0;
280  let Inst{3-0}   = Pd;
281
282  let Defs = !if(!eq (opc{0}, 1), [NZCV], []);
283  let ElementSize = pprty.ElementSize;
284  let isReMaterializable = 1;
285}
286
287multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> {
288  def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>;
289  def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>;
290  def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>;
291  def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>;
292
293  def : InstAlias<asm # "\t$Pd",
294                  (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>;
295  def : InstAlias<asm # "\t$Pd",
296                  (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>;
297  def : InstAlias<asm # "\t$Pd",
298                  (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>;
299  def : InstAlias<asm # "\t$Pd",
300                  (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>;
301}
302
303def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>;
304def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>;
305
306let Predicates = [HasSVE] in {
307  defm PTRUE  : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>;
308  defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>;
309}
310
311//===----------------------------------------------------------------------===//
312// SVE pattern match helpers.
313//===----------------------------------------------------------------------===//
314
315class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
316                   Instruction inst>
317: Pat<(vtd (op vt1:$Op1)),
318      (inst $Op1)>;
319
320class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
321                            ValueType vts, Instruction inst>
322: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)),
323      (inst $Op3, $Op1, $Op2)>;
324
325// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the
326// type of rounding. This is matched by timm0_1 in pattern below and ignored.
327class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
328                                  ValueType vts, Instruction inst>
329: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)),
330      (inst $Op3, $Op1, $Op2)>;
331
332class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
333                                      ValueType it, ComplexPattern cpx, Instruction inst>
334  : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))),
335        (inst $Op1, i32:$imm, i32:$shift)>;
336
337class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
338                              ValueType it, ComplexPattern cpx, Instruction inst>
339  : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))),
340        (inst $Op1, i32:$imm, i32:$shift)>;
341
342class SVE_1_Op_Imm_Arith_Pred_Pat<ValueType vt, ValueType pt, SDPatternOperator op,
343                                  ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst>
344  : Pat<(vt (op (pt (AArch64ptrue 31)), (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))),
345        (inst $Op1, i32:$imm)>;
346
347class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
348                           ValueType it, ComplexPattern cpx, Instruction inst>
349  : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))),
350        (inst $Op1, i64:$imm)>;
351
352class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
353                   ValueType vt2, Instruction inst>
354: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),
355      (inst $Op1, $Op2)>;
356
357class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op,
358                               ValueType pt, ValueType vt1, ValueType vt2,
359                               Instruction inst>
360: Pat<(vtd (op (pt (AArch64ptrue 31)), vt1:$Op1, vt2:$Op2)),
361      (inst $Op1, $Op2)>;
362
363class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
364                   ValueType vt2, ValueType vt3, Instruction inst>
365: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)),
366      (inst $Op1, $Op2, $Op3)>;
367
368class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
369                   ValueType vt2, ValueType vt3, ValueType vt4,
370                   Instruction inst>
371: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)),
372      (inst $Op1, $Op2, $Op3, $Op4)>;
373
374class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
375                       ValueType vt2, Operand ImmTy, Instruction inst>
376: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))),
377      (inst $Op1, ImmTy:$Op2)>;
378
379class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
380                       ValueType vt2, ValueType vt3, Operand ImmTy,
381                       Instruction inst>
382: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))),
383      (inst $Op1, $Op2, ImmTy:$Op3)>;
384
385class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
386                       ValueType vt2, ValueType vt3, ValueType vt4,
387                       Operand ImmTy, Instruction inst>
388: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))),
389      (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>;
390
391def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>;
392def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>;
393
394let AddedComplexity = 1 in {
395class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1,
396                   ValueType vt2, ValueType vt3, Instruction inst>
397: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))),
398      (inst $Op1, $Op2, $Op3)>;
399
400class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op,
401                                     ValueType vt1, ValueType vt2,
402                                     Operand vt3, Instruction inst>
403: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))),
404      (inst $Op1, $Op2, vt3:$Op3)>;
405}
406
407//
408// Common but less generic patterns.
409//
410
411class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
412                             Instruction inst, Instruction ptrue>
413: Pat<(vtd (op vt1:$Op1)),
414      (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>;
415
416class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
417                             ValueType vt2, Instruction inst, Instruction ptrue>
418: Pat<(vtd (op vt1:$Op1, vt2:$Op2)),
419      (inst (ptrue 31), $Op1, $Op2)>;
420
421class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt,
422                       ValueType inreg_vt, Instruction inst>
423: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)),
424      (inst $PassThru, $Pg, $Src)>;
425
426class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op,
427                                ValueType pt, ValueType it,
428                                ComplexPattern cast, Instruction inst>
429: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))),
430      (inst $Pg, $Rn, i32:$imm)>;
431
432class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op,
433                                      ValueType pt, ValueType it,
434                                      ComplexPattern cast, Instruction inst>
435: Pat<(vt (op (pt (AArch64ptrue 31)), vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))),
436      (inst $Rn, i32:$imm)>;
437
438//
439// Pseudo -> Instruction mappings
440//
441def getSVEPseudoMap : InstrMapping {
442  let FilterClass = "SVEPseudo2Instr";
443  let RowFields = ["PseudoName"];
444  let ColFields = ["IsInstr"];
445  let KeyCol = ["0"];
446  let ValueCols = [["1"]];
447}
448
449class SVEPseudo2Instr<string name, bit instr> {
450  string PseudoName = name;
451  bit IsInstr = instr;
452}
453
454// Lookup e.g. DIV -> DIVR
455def getSVERevInstr : InstrMapping {
456  let FilterClass = "SVEInstr2Rev";
457  let RowFields = ["InstrName"];
458  let ColFields = ["isReverseInstr"];
459  let KeyCol = ["0"];
460  let ValueCols = [["1"]];
461}
462
463// Lookup e.g. DIVR -> DIV
464def getSVENonRevInstr : InstrMapping {
465  let FilterClass = "SVEInstr2Rev";
466  let RowFields = ["InstrName"];
467  let ColFields = ["isReverseInstr"];
468  let KeyCol = ["1"];
469  let ValueCols = [["0"]];
470}
471
472class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> {
473  string InstrName = !if(name1IsReverseInstr, name1, name2);
474  bit isReverseInstr = name1IsReverseInstr;
475}
476
477//
478// Pseudos for destructive operands
479//
480let hasNoSchedulingInfo = 1 in {
481  class PredTwoOpPseudo<string name, ZPRRegOp zprty,
482                        FalseLanesEnum flags = FalseLanesNone>
483  : SVEPseudo2Instr<name, 0>,
484    Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> {
485    let FalseLanes = flags;
486  }
487
488  class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty,
489                           FalseLanesEnum flags = FalseLanesNone>
490  : SVEPseudo2Instr<name, 0>,
491    Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> {
492    let FalseLanes = flags;
493  }
494}
495
496//===----------------------------------------------------------------------===//
497// SVE Predicate Misc Group
498//===----------------------------------------------------------------------===//
499
500class sve_int_pfalse<bits<6> opc, string asm>
501: I<(outs PPR8:$Pd), (ins),
502  asm, "\t$Pd",
503  "",
504  []>, Sched<[]> {
505  bits<4> Pd;
506  let Inst{31-24} = 0b00100101;
507  let Inst{23-22} = opc{5-4};
508  let Inst{21-19} = 0b011;
509  let Inst{18-16} = opc{3-1};
510  let Inst{15-10} = 0b111001;
511  let Inst{9}     = opc{0};
512  let Inst{8-4}   = 0b00000;
513  let Inst{3-0}   = Pd;
514
515  let isReMaterializable = 1;
516}
517
518class sve_int_ptest<bits<6> opc, string asm>
519: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn),
520  asm, "\t$Pg, $Pn",
521  "",
522  []>, Sched<[]> {
523  bits<4> Pg;
524  bits<4> Pn;
525  let Inst{31-24} = 0b00100101;
526  let Inst{23-22} = opc{5-4};
527  let Inst{21-19} = 0b010;
528  let Inst{18-16} = opc{3-1};
529  let Inst{15-14} = 0b11;
530  let Inst{13-10} = Pg;
531  let Inst{9}     = opc{0};
532  let Inst{8-5}   = Pn;
533  let Inst{4-0}   = 0b00000;
534
535  let Defs = [NZCV];
536  let isCompare = 1;
537}
538
539class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm,
540                          PPRRegOp pprty>
541: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn),
542  asm, "\t$Pdn, $Pg, $_Pdn",
543  "",
544  []>, Sched<[]> {
545  bits<4> Pdn;
546  bits<4> Pg;
547  let Inst{31-24} = 0b00100101;
548  let Inst{23-22} = sz8_64;
549  let Inst{21-19} = 0b011;
550  let Inst{18-16} = opc{4-2};
551  let Inst{15-11} = 0b11000;
552  let Inst{10-9}  = opc{1-0};
553  let Inst{8-5}   = Pg;
554  let Inst{4}     = 0;
555  let Inst{3-0}   = Pdn;
556
557  let Constraints = "$Pdn = $_Pdn";
558  let Defs = [NZCV];
559}
560
561multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> {
562  def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>;
563
564  def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
565}
566
567multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> {
568  def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>;
569  def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>;
570  def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>;
571  def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>;
572
573  def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
574  def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>;
575  def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>;
576  def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>;
577}
578
579//===----------------------------------------------------------------------===//
580// SVE Predicate Count Group
581//===----------------------------------------------------------------------===//
582
583class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm,
584                      RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty>
585: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn),
586  asm, "\t$Rdn, $Pg",
587  "",
588  []>, Sched<[]> {
589  bits<5> Rdn;
590  bits<4> Pg;
591  let Inst{31-24} = 0b00100101;
592  let Inst{23-22} = sz8_64;
593  let Inst{21-19} = 0b101;
594  let Inst{18-16} = opc{4-2};
595  let Inst{15-11} = 0b10001;
596  let Inst{10-9}  = opc{1-0};
597  let Inst{8-5}   = Pg;
598  let Inst{4-0}   = Rdn;
599
600  // Signed 32bit forms require their GPR operand printed.
601  let AsmString = !if(!eq(opc{4,2-0}, 0b0000),
602                      !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"),
603                      !strconcat(asm, "\t$Rdn, $Pg"));
604  let Constraints = "$Rdn = $_Rdn";
605}
606
607multiclass sve_int_count_r_s32<bits<5> opc, string asm,
608                               SDPatternOperator op> {
609  def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>;
610  def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>;
611  def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>;
612  def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>;
613
614  def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))),
615            (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
616  def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))),
617            (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
618
619  def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))),
620            (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
621  def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))),
622            (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
623
624  def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))),
625            (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
626  def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))),
627            (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
628
629  def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))),
630            (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>;
631  def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))),
632            (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>;
633}
634
635multiclass sve_int_count_r_u32<bits<5> opc, string asm,
636                               SDPatternOperator op> {
637  def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>;
638  def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>;
639  def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>;
640  def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>;
641
642  def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))),
643            (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>;
644  def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))),
645            (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>;
646  def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))),
647            (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>;
648  def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))),
649            (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>;
650}
651
652multiclass sve_int_count_r_x64<bits<5> opc, string asm,
653                               SDPatternOperator op = null_frag> {
654  def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>;
655  def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>;
656  def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>;
657  def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>;
658
659  def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))),
660            (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>;
661  def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))),
662            (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>;
663  def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))),
664            (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>;
665  def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))),
666            (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>;
667}
668
669class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm,
670                      ZPRRegOp zprty, PPRRegOp pprty>
671: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm),
672  asm, "\t$Zdn, $Pm",
673  "",
674  []>, Sched<[]> {
675  bits<4> Pm;
676  bits<5> Zdn;
677  let Inst{31-24} = 0b00100101;
678  let Inst{23-22} = sz8_64;
679  let Inst{21-19} = 0b101;
680  let Inst{18-16} = opc{4-2};
681  let Inst{15-11} = 0b10000;
682  let Inst{10-9}  = opc{1-0};
683  let Inst{8-5}   = Pm;
684  let Inst{4-0}   = Zdn;
685
686  let Constraints = "$Zdn = $_Zdn";
687  let DestructiveInstType = DestructiveOther;
688  let ElementSize = ElementSizeNone;
689}
690
691multiclass sve_int_count_v<bits<5> opc, string asm,
692                           SDPatternOperator op = null_frag> {
693  def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>;
694  def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>;
695  def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>;
696
697  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16,  nxv8i1, !cast<Instruction>(NAME # _H)>;
698  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32,  nxv4i1, !cast<Instruction>(NAME # _S)>;
699  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64,  nxv2i1, !cast<Instruction>(NAME # _D)>;
700
701  def : InstAlias<asm # "\t$Zdn, $Pm",
702                 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>;
703  def : InstAlias<asm # "\t$Zdn, $Pm",
704                 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>;
705  def : InstAlias<asm # "\t$Zdn, $Pm",
706                  (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>;
707}
708
709class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm,
710                          PPRRegOp pprty>
711: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn),
712  asm, "\t$Rd, $Pg, $Pn",
713  "",
714  []>, Sched<[]> {
715  bits<4> Pg;
716  bits<4> Pn;
717  bits<5> Rd;
718  let Inst{31-24} = 0b00100101;
719  let Inst{23-22} = sz8_64;
720  let Inst{21-19} = 0b100;
721  let Inst{18-16} = opc{3-1};
722  let Inst{15-14} = 0b10;
723  let Inst{13-10} = Pg;
724  let Inst{9}     = opc{0};
725  let Inst{8-5}   = Pn;
726  let Inst{4-0}   = Rd;
727}
728
729multiclass sve_int_pcount_pred<bits<4> opc, string asm,
730                               SDPatternOperator int_op> {
731  def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>;
732  def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>;
733  def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>;
734  def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>;
735
736  def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
737  def : SVE_2_Op_Pat<i64, int_op, nxv8i1,  nxv8i1,  !cast<Instruction>(NAME # _H)>;
738  def : SVE_2_Op_Pat<i64, int_op, nxv4i1,  nxv4i1,  !cast<Instruction>(NAME # _S)>;
739  def : SVE_2_Op_Pat<i64, int_op, nxv2i1,  nxv2i1,  !cast<Instruction>(NAME # _D)>;
740}
741
742//===----------------------------------------------------------------------===//
743// SVE Element Count Group
744//===----------------------------------------------------------------------===//
745
746class sve_int_count<bits<3> opc, string asm>
747: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
748  asm, "\t$Rd, $pattern, mul $imm4",
749  "",
750  []>, Sched<[]> {
751  bits<5> Rd;
752  bits<4> imm4;
753  bits<5> pattern;
754  let Inst{31-24} = 0b00000100;
755  let Inst{23-22} = opc{2-1};
756  let Inst{21-20} = 0b10;
757  let Inst{19-16} = imm4;
758  let Inst{15-11} = 0b11100;
759  let Inst{10}    = opc{0};
760  let Inst{9-5}   = pattern;
761  let Inst{4-0}   = Rd;
762}
763
764multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> {
765  def NAME : sve_int_count<opc, asm>;
766
767  def : InstAlias<asm # "\t$Rd, $pattern",
768                  (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>;
769  def : InstAlias<asm # "\t$Rd",
770                  (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>;
771
772  def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))),
773            (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;
774
775  def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))),
776            (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>;
777
778  def : Pat<(i64 (op sve_pred_enum:$pattern)),
779            (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>;
780}
781
782class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty>
783: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
784  asm, "\t$Zdn, $pattern, mul $imm4",
785  "",
786  []>, Sched<[]> {
787  bits<5> Zdn;
788  bits<5> pattern;
789  bits<4> imm4;
790  let Inst{31-24} = 0b00000100;
791  let Inst{23-22} = opc{4-3};
792  let Inst{21}    = 0b1;
793  let Inst{20}    = opc{2};
794  let Inst{19-16} = imm4;
795  let Inst{15-12} = 0b1100;
796  let Inst{11-10} = opc{1-0};
797  let Inst{9-5}   = pattern;
798  let Inst{4-0}   = Zdn;
799
800  let Constraints = "$Zdn = $_Zdn";
801  let DestructiveInstType = DestructiveOther;
802  let ElementSize = ElementSizeNone;
803}
804
805multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty,
806                            SDPatternOperator op = null_frag,
807                            ValueType vt = OtherVT> {
808  def NAME : sve_int_countvlv<opc, asm, zprty>;
809
810  def : InstAlias<asm # "\t$Zdn, $pattern",
811                  (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>;
812  def : InstAlias<asm # "\t$Zdn",
813                  (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>;
814
815  def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
816            (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
817}
818
819class sve_int_pred_pattern_a<bits<3> opc, string asm>
820: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
821  asm, "\t$Rdn, $pattern, mul $imm4",
822  "",
823  []>, Sched<[]> {
824  bits<5> Rdn;
825  bits<5> pattern;
826  bits<4> imm4;
827  let Inst{31-24} = 0b00000100;
828  let Inst{23-22} = opc{2-1};
829  let Inst{21-20} = 0b11;
830  let Inst{19-16} = imm4;
831  let Inst{15-11} = 0b11100;
832  let Inst{10}    = opc{0};
833  let Inst{9-5}   = pattern;
834  let Inst{4-0}   = Rdn;
835
836  let Constraints = "$Rdn = $_Rdn";
837}
838
839multiclass sve_int_pred_pattern_a<bits<3> opc, string asm> {
840  def NAME : sve_int_pred_pattern_a<opc, asm>;
841
842  def : InstAlias<asm # "\t$Rdn, $pattern",
843                  (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>;
844  def : InstAlias<asm # "\t$Rdn",
845                  (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>;
846}
847
848class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt,
849                             RegisterOperand st>
850: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4),
851  asm, "\t$Rdn, $pattern, mul $imm4",
852  "",
853  []>, Sched<[]> {
854  bits<5> Rdn;
855  bits<5> pattern;
856  bits<4> imm4;
857  let Inst{31-24} = 0b00000100;
858  let Inst{23-22} = opc{4-3};
859  let Inst{21}    = 0b1;
860  let Inst{20}    = opc{2};
861  let Inst{19-16} = imm4;
862  let Inst{15-12} = 0b1111;
863  let Inst{11-10} = opc{1-0};
864  let Inst{9-5}   = pattern;
865  let Inst{4-0}   = Rdn;
866
867  // Signed 32bit forms require their GPR operand printed.
868  let AsmString = !if(!eq(opc{2,0}, 0b00),
869                      !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"),
870                      !strconcat(asm, "\t$Rdn, $pattern, mul $imm4"));
871
872  let Constraints = "$Rdn = $_Rdn";
873}
874
875multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm,
876                                      SDPatternOperator op> {
877  def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>;
878
879  def : InstAlias<asm # "\t$Rd, $Rn, $pattern",
880                  (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>;
881  def : InstAlias<asm # "\t$Rd, $Rn",
882                  (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>;
883
884  // NOTE: Register allocation doesn't like tied operands of differing register
885  //       class, hence the extra INSERT_SUBREG complication.
886
887  def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
888            (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>;
889  def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))),
890            (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
891}
892
893multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm,
894                                      SDPatternOperator op> {
895  def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>;
896
897  def : InstAlias<asm # "\t$Rdn, $pattern",
898                  (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>;
899  def : InstAlias<asm # "\t$Rdn",
900                  (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>;
901
902  def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
903            (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
904}
905
906multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm,
907                                      SDPatternOperator op> {
908  def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>;
909
910  def : InstAlias<asm # "\t$Rdn, $pattern",
911                  (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>;
912  def : InstAlias<asm # "\t$Rdn",
913                  (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>;
914
915  def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))),
916            (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>;
917}
918
919
920//===----------------------------------------------------------------------===//
921// SVE Permute - Cross Lane Group
922//===----------------------------------------------------------------------===//
923
924class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,
925                         ValueType vt, RegisterClass srcRegType,
926                         SDPatternOperator op>
927: I<(outs zprty:$Zd), (ins srcRegType:$Rn),
928  asm, "\t$Zd, $Rn",
929  "",
930  [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> {
931  bits<5> Rn;
932  bits<5> Zd;
933  let Inst{31-24} = 0b00000101;
934  let Inst{23-22} = sz8_64;
935  let Inst{21-10} = 0b100000001110;
936  let Inst{9-5}   = Rn;
937  let Inst{4-0}   = Zd;
938}
939
940multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> {
941  def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>;
942  def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>;
943  def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>;
944  def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>;
945
946  def : InstAlias<"mov $Zd, $Rn",
947                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>;
948  def : InstAlias<"mov $Zd, $Rn",
949                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>;
950  def : InstAlias<"mov $Zd, $Rn",
951                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>;
952  def : InstAlias<"mov $Zd, $Rn",
953                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>;
954}
955
956class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm,
957                         ZPRRegOp zprty>
958: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx),
959  asm, "\t$Zd, $Zn$idx",
960  "",
961  []>, Sched<[]> {
962  bits<5> Zd;
963  bits<5> Zn;
964  bits<7> idx;
965  let Inst{31-24} = 0b00000101;
966  let Inst{23-22} = {?,?}; // imm3h
967  let Inst{21}    = 0b1;
968  let Inst{20-16} = tsz;
969  let Inst{15-10} = 0b001000;
970  let Inst{9-5}   = Zn;
971  let Inst{4-0}   = Zd;
972}
973
974multiclass sve_int_perm_dup_i<string asm> {
975  def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> {
976    let Inst{23-22} = idx{5-4};
977    let Inst{20-17} = idx{3-0};
978  }
979  def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> {
980    let Inst{23-22} = idx{4-3};
981    let Inst{20-18} = idx{2-0};
982  }
983  def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> {
984    let Inst{23-22} = idx{3-2};
985    let Inst{20-19}    = idx{1-0};
986  }
987  def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> {
988    let Inst{23-22} = idx{2-1};
989    let Inst{20}    = idx{0};
990  }
991  def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> {
992    let Inst{23-22} = idx{1-0};
993  }
994
995  def : InstAlias<"mov $Zd, $Zn$idx",
996                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>;
997  def : InstAlias<"mov $Zd, $Zn$idx",
998                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>;
999  def : InstAlias<"mov $Zd, $Zn$idx",
1000                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>;
1001  def : InstAlias<"mov $Zd, $Zn$idx",
1002                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>;
1003  def : InstAlias<"mov $Zd, $Zn$idx",
1004                  (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>;
1005  def : InstAlias<"mov $Zd, $Bn",
1006                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>;
1007  def : InstAlias<"mov $Zd, $Hn",
1008                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>;
1009  def : InstAlias<"mov $Zd, $Sn",
1010                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>;
1011  def : InstAlias<"mov $Zd, $Dn",
1012                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>;
1013  def : InstAlias<"mov $Zd, $Qn",
1014                  (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>;
1015}
1016
1017class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty,
1018                       RegisterOperand VecList>
1019: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm),
1020  asm, "\t$Zd, $Zn, $Zm",
1021  "",
1022  []>, Sched<[]> {
1023  bits<5> Zd;
1024  bits<5> Zm;
1025  bits<5> Zn;
1026  let Inst{31-24} = 0b00000101;
1027  let Inst{23-22} = sz8_64;
1028  let Inst{21}    = 0b1;
1029  let Inst{20-16} = Zm;
1030  let Inst{15-13} = 0b001;
1031  let Inst{12-11} = opc;
1032  let Inst{10}    = 0b0;
1033  let Inst{9-5}   = Zn;
1034  let Inst{4-0}   = Zd;
1035}
1036
1037multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> {
1038  def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8,  Z_b>;
1039  def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>;
1040  def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>;
1041  def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>;
1042
1043  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
1044                 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>;
1045  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
1046                 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>;
1047  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
1048                 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>;
1049  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
1050                 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>;
1051
1052  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
1053  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1054  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1055  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1056
1057  def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1058  def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1059  def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1060
1061  def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1062}
1063
1064multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> {
1065  def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8,  ZZ_b>;
1066  def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>;
1067  def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>;
1068  def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>;
1069
1070  def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)),
1071            (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0,
1072                                                                        nxv16i8:$Op2, zsub1),
1073                                                     nxv16i8:$Op3))>;
1074
1075  def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)),
1076            (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0,
1077                                                                        nxv8i16:$Op2, zsub1),
1078                                                     nxv8i16:$Op3))>;
1079
1080  def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)),
1081            (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0,
1082                                                                        nxv4i32:$Op2, zsub1),
1083                                                     nxv4i32:$Op3))>;
1084
1085  def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)),
1086            (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0,
1087                                                                        nxv2i64:$Op2, zsub1),
1088                                                     nxv2i64:$Op3))>;
1089
1090  def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)),
1091            (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0,
1092                                                                        nxv8f16:$Op2, zsub1),
1093                                                     nxv8i16:$Op3))>;
1094
1095  def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)),
1096            (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0,
1097                                                                        nxv4f32:$Op2, zsub1),
1098                                                     nxv4i32:$Op3))>;
1099
1100  def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)),
1101            (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0,
1102                                                                        nxv2f64:$Op2, zsub1),
1103                                                     nxv2i64:$Op3))>;
1104
1105  def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)),
1106            (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0,
1107                                                                         nxv8bf16:$Op2, zsub1),
1108                                                      nxv8i16:$Op3))>;
1109}
1110
1111class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty>
1112: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm),
1113  asm, "\t$Zd, $Zn, $Zm",
1114  "",
1115  []>, Sched<[]> {
1116  bits<5> Zd;
1117  bits<5> Zm;
1118  bits<5> Zn;
1119  let Inst{31-24} = 0b00000101;
1120  let Inst{23-22} = sz8_64;
1121  let Inst{21}    = 0b1;
1122  let Inst{20-16} = Zm;
1123  let Inst{15-10} = 0b001011;
1124  let Inst{9-5}   = Zn;
1125  let Inst{4-0}   = Zd;
1126
1127  let Constraints = "$Zd = $_Zd";
1128}
1129
1130multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> {
1131  def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>;
1132  def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>;
1133  def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>;
1134  def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>;
1135
1136  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
1137  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1138  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1139  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1140
1141  def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1142  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1143  def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1144
1145  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1146}
1147
1148class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty>
1149: I<(outs zprty:$Zd), (ins zprty:$Zn),
1150  asm, "\t$Zd, $Zn",
1151  "",
1152  []>, Sched<[]> {
1153  bits<5> Zd;
1154  bits<5> Zn;
1155  let Inst{31-24} = 0b00000101;
1156  let Inst{23-22} = sz8_64;
1157  let Inst{21-10} = 0b111000001110;
1158  let Inst{9-5}   = Zn;
1159  let Inst{4-0}   = Zd;
1160}
1161
1162multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> {
1163  def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>;
1164  def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>;
1165  def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>;
1166  def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>;
1167
1168  def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>;
1169  def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>;
1170  def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>;
1171  def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>;
1172
1173  def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>;
1174  def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>;
1175  def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>;
1176
1177  def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>;
1178}
1179
1180class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty>
1181: I<(outs pprty:$Pd), (ins pprty:$Pn),
1182  asm, "\t$Pd, $Pn",
1183  "",
1184  []>, Sched<[]> {
1185  bits<4> Pd;
1186  bits<4> Pn;
1187  let Inst{31-24} = 0b00000101;
1188  let Inst{23-22} = sz8_64;
1189  let Inst{21-9}  = 0b1101000100000;
1190  let Inst{8-5}   = Pn;
1191  let Inst{4}     = 0b0;
1192  let Inst{3-0}   = Pd;
1193}
1194
1195multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> {
1196  def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>;
1197  def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>;
1198  def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>;
1199  def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>;
1200
1201  def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>;
1202  def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>;
1203  def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>;
1204  def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>;
1205}
1206
1207class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm,
1208                        ZPRRegOp zprty1, ZPRRegOp zprty2>
1209: I<(outs zprty1:$Zd), (ins zprty2:$Zn),
1210  asm, "\t$Zd, $Zn",
1211  "", []>, Sched<[]> {
1212  bits<5> Zd;
1213  bits<5> Zn;
1214  let Inst{31-24} = 0b00000101;
1215  let Inst{23-22} = sz16_64;
1216  let Inst{21-18} = 0b1100;
1217  let Inst{17-16} = opc;
1218  let Inst{15-10} = 0b001110;
1219  let Inst{9-5}   = Zn;
1220  let Inst{4-0}   = Zd;
1221}
1222
1223multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> {
1224  def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>;
1225  def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>;
1226  def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>;
1227
1228  def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>;
1229  def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>;
1230  def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>;
1231}
1232
1233class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty,
1234                         RegisterClass srcRegType>
1235: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm),
1236  asm, "\t$Zdn, $Rm",
1237  "",
1238  []>, Sched<[]> {
1239  bits<5> Rm;
1240  bits<5> Zdn;
1241  let Inst{31-24} = 0b00000101;
1242  let Inst{23-22} = sz8_64;
1243  let Inst{21-10} = 0b100100001110;
1244  let Inst{9-5}   = Rm;
1245  let Inst{4-0}   = Zdn;
1246
1247  let Constraints = "$Zdn = $_Zdn";
1248  let DestructiveInstType = DestructiveOther;
1249}
1250
1251multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> {
1252  def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>;
1253  def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>;
1254  def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>;
1255  def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>;
1256
1257  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>;
1258  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>;
1259  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>;
1260  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>;
1261}
1262
1263class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty,
1264                         RegisterClass srcRegType>
1265: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Vm),
1266  asm, "\t$Zdn, $Vm",
1267  "",
1268  []>, Sched<[]> {
1269  bits<5> Vm;
1270  bits<5> Zdn;
1271  let Inst{31-24} = 0b00000101;
1272  let Inst{23-22} = sz8_64;
1273  let Inst{21-10} = 0b110100001110;
1274  let Inst{9-5}   = Vm;
1275  let Inst{4-0}   = Zdn;
1276
1277  let Constraints = "$Zdn = $_Zdn";
1278  let DestructiveInstType = DestructiveOther;
1279}
1280
1281multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> {
1282  def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8>;
1283  def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16>;
1284  def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32>;
1285  def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64>;
1286
1287  def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, f16, !cast<Instruction>(NAME # _H)>;
1288  def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, f32, !cast<Instruction>(NAME # _S)>;
1289  def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, f64, !cast<Instruction>(NAME # _D)>;
1290
1291  def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, bf16, !cast<Instruction>(NAME # _H)>;
1292}
1293
1294//===----------------------------------------------------------------------===//
1295// SVE Permute - Extract Group
1296//===----------------------------------------------------------------------===//
1297
1298class sve_int_perm_extract_i<string asm>
1299: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8),
1300  asm, "\t$Zdn, $_Zdn, $Zm, $imm8",
1301  "", []>, Sched<[]> {
1302  bits<5> Zdn;
1303  bits<5> Zm;
1304  bits<8> imm8;
1305  let Inst{31-21} = 0b00000101001;
1306  let Inst{20-16} = imm8{7-3};
1307  let Inst{15-13} = 0b000;
1308  let Inst{12-10} = imm8{2-0};
1309  let Inst{9-5}   = Zm;
1310  let Inst{4-0}   = Zdn;
1311
1312  let Constraints = "$Zdn = $_Zdn";
1313  let DestructiveInstType = DestructiveOther;
1314  let ElementSize = ElementSizeNone;
1315}
1316
1317multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> {
1318  def NAME : sve_int_perm_extract_i<asm>;
1319
1320  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255,
1321                         !cast<Instruction>(NAME)>;
1322}
1323
1324class sve2_int_perm_extract_i_cons<string asm>
1325: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8),
1326  asm, "\t$Zd, $Zn, $imm8",
1327  "", []>, Sched<[]> {
1328  bits<5> Zd;
1329  bits<5> Zn;
1330  bits<8> imm8;
1331  let Inst{31-21} = 0b00000101011;
1332  let Inst{20-16} = imm8{7-3};
1333  let Inst{15-13} = 0b000;
1334  let Inst{12-10} = imm8{2-0};
1335  let Inst{9-5}   = Zn;
1336  let Inst{4-0}   = Zd;
1337}
1338
1339//===----------------------------------------------------------------------===//
1340// SVE Vector Select Group
1341//===----------------------------------------------------------------------===//
1342
1343class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty>
1344: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm),
1345  asm, "\t$Zd, $Pg, $Zn, $Zm",
1346  "",
1347  []>, Sched<[]> {
1348  bits<4> Pg;
1349  bits<5> Zd;
1350  bits<5> Zm;
1351  bits<5> Zn;
1352  let Inst{31-24} = 0b00000101;
1353  let Inst{23-22} = sz8_64;
1354  let Inst{21}    = 0b1;
1355  let Inst{20-16} = Zm;
1356  let Inst{15-14} = 0b11;
1357  let Inst{13-10} = Pg;
1358  let Inst{9-5}   = Zn;
1359  let Inst{4-0}   = Zd;
1360}
1361
1362multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> {
1363  def _B : sve_int_sel_vvv<0b00, asm, ZPR8>;
1364  def _H : sve_int_sel_vvv<0b01, asm, ZPR16>;
1365  def _S : sve_int_sel_vvv<0b10, asm, ZPR32>;
1366  def _D : sve_int_sel_vvv<0b11, asm, ZPR64>;
1367
1368  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
1369  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1370  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1371  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1372
1373  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1,  nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
1374  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1,  nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
1375  def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1,  nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>;
1376  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1,  nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
1377
1378  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1,  nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
1379
1380  def : InstAlias<"mov $Zd, $Pg/m, $Zn",
1381                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>;
1382  def : InstAlias<"mov $Zd, $Pg/m, $Zn",
1383                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>;
1384  def : InstAlias<"mov $Zd, $Pg/m, $Zn",
1385                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>;
1386  def : InstAlias<"mov $Zd, $Pg/m, $Zn",
1387                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>;
1388}
1389
1390
1391//===----------------------------------------------------------------------===//
1392// SVE Predicate Logical Operations Group
1393//===----------------------------------------------------------------------===//
1394
1395class sve_int_pred_log<bits<4> opc, string asm>
1396: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm),
1397  asm, "\t$Pd, $Pg/z, $Pn, $Pm",
1398  "",
1399  []>, Sched<[]> {
1400  bits<4> Pd;
1401  bits<4> Pg;
1402  bits<4> Pm;
1403  bits<4> Pn;
1404  let Inst{31-24} = 0b00100101;
1405  let Inst{23-22} = opc{3-2};
1406  let Inst{21-20} = 0b00;
1407  let Inst{19-16} = Pm;
1408  let Inst{15-14} = 0b01;
1409  let Inst{13-10} = Pg;
1410  let Inst{9}     = opc{1};
1411  let Inst{8-5}   = Pn;
1412  let Inst{4}     = opc{0};
1413  let Inst{3-0}   = Pd;
1414
1415  // SEL has no predication qualifier.
1416  let AsmString = !if(!eq(opc, 0b0011),
1417                      !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"),
1418                      !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm"));
1419
1420  let Defs = !if(!eq (opc{2}, 1), [NZCV], []);
1421
1422}
1423
1424multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op,
1425                            SDPatternOperator op_nopred = null_frag> {
1426  def NAME : sve_int_pred_log<opc, asm>;
1427
1428  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
1429  def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>;
1430  def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>;
1431  def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>;
1432  def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1,
1433                               !cast<Instruction>(NAME), PTRUE_B>;
1434  def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1,
1435                               !cast<Instruction>(NAME), PTRUE_H>;
1436  def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1,
1437                               !cast<Instruction>(NAME), PTRUE_S>;
1438  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1,
1439                               !cast<Instruction>(NAME), PTRUE_D>;
1440}
1441
1442//===----------------------------------------------------------------------===//
1443// SVE Logical Mask Immediate Group
1444//===----------------------------------------------------------------------===//
1445
1446class sve_int_log_imm<bits<2> opc, string asm>
1447: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13),
1448  asm, "\t$Zdn, $_Zdn, $imms13",
1449  "", []>, Sched<[]> {
1450  bits<5> Zdn;
1451  bits<13> imms13;
1452  let Inst{31-24} = 0b00000101;
1453  let Inst{23-22} = opc;
1454  let Inst{21-18} = 0b0000;
1455  let Inst{17-5}  = imms13;
1456  let Inst{4-0}   = Zdn;
1457
1458  let Constraints = "$Zdn = $_Zdn";
1459  let DecoderMethod = "DecodeSVELogicalImmInstruction";
1460  let DestructiveInstType = DestructiveOther;
1461  let ElementSize = ElementSizeNone;
1462}
1463
1464multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> {
1465  def NAME : sve_int_log_imm<opc, asm>;
1466
1467  def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8,  i32, SVELogicalImm8Pat,  !cast<Instruction>(NAME)>;
1468  def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>;
1469  def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>;
1470  def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>;
1471
1472  def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
1473                  (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>;
1474  def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
1475                  (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>;
1476  def : InstAlias<asm # "\t$Zdn, $Zdn, $imm",
1477                  (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>;
1478
1479  def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
1480                  (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>;
1481  def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
1482                  (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>;
1483  def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
1484                  (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>;
1485  def : InstAlias<alias # "\t$Zdn, $Zdn, $imm",
1486                  (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>;
1487}
1488
1489class sve_int_dup_mask_imm<string asm>
1490: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms),
1491  asm, "\t$Zd, $imms",
1492  "",
1493  []>, Sched<[]> {
1494  bits<5> Zd;
1495  bits<13> imms;
1496  let Inst{31-18} = 0b00000101110000;
1497  let Inst{17-5} = imms;
1498  let Inst{4-0} = Zd;
1499
1500  let isReMaterializable = 1;
1501  let DecoderMethod = "DecodeSVELogicalImmInstruction";
1502}
1503
1504multiclass sve_int_dup_mask_imm<string asm> {
1505  def NAME : sve_int_dup_mask_imm<asm>;
1506
1507  def : InstAlias<"dupm $Zd, $imm",
1508                  (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>;
1509  def : InstAlias<"dupm $Zd, $imm",
1510                  (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>;
1511  def : InstAlias<"dupm $Zd, $imm",
1512                  (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>;
1513
1514  // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here.
1515  def : InstAlias<"mov $Zd, $imm",
1516                  (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>;
1517  def : InstAlias<"mov $Zd, $imm",
1518                  (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>;
1519  def : InstAlias<"mov $Zd, $imm",
1520                  (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>;
1521}
1522
1523//===----------------------------------------------------------------------===//
1524// SVE Integer Arithmetic -  Unpredicated Group.
1525//===----------------------------------------------------------------------===//
1526
1527class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm,
1528                              ZPRRegOp zprty>
1529: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
1530  asm, "\t$Zd, $Zn, $Zm",
1531  "", []>, Sched<[]> {
1532  bits<5> Zd;
1533  bits<5> Zm;
1534  bits<5> Zn;
1535  let Inst{31-24} = 0b00000100;
1536  let Inst{23-22} = sz8_64;
1537  let Inst{21}    = 0b1;
1538  let Inst{20-16} = Zm;
1539  let Inst{15-13} = 0b000;
1540  let Inst{12-10} = opc;
1541  let Inst{9-5}   = Zn;
1542  let Inst{4-0}   = Zd;
1543}
1544
1545multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm,
1546                                   SDPatternOperator op, SDPatternOperator int_op> {
1547  def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>;
1548  def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>;
1549  def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>;
1550  def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>;
1551
1552  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
1553  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1554  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1555  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1556
1557  // Intrinsic version
1558  def : SVE_2_Op_Pat<nxv16i8, int_op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
1559  def : SVE_2_Op_Pat<nxv8i16, int_op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1560  def : SVE_2_Op_Pat<nxv4i32, int_op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1561  def : SVE_2_Op_Pat<nxv2i64, int_op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1562}
1563
1564//===----------------------------------------------------------------------===//
1565// SVE Floating Point Arithmetic - Predicated Group
1566//===----------------------------------------------------------------------===//
1567
1568class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm,
1569                         ZPRRegOp zprty,
1570                         Operand imm_ty>
1571: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1),
1572  asm, "\t$Zdn, $Pg/m, $_Zdn, $i1",
1573  "",
1574  []>, Sched<[]> {
1575  bits<3> Pg;
1576  bits<5> Zdn;
1577  bit i1;
1578  let Inst{31-24} = 0b01100101;
1579  let Inst{23-22} = sz;
1580  let Inst{21-19} = 0b011;
1581  let Inst{18-16} = opc;
1582  let Inst{15-13} = 0b100;
1583  let Inst{12-10} = Pg;
1584  let Inst{9-6}   = 0b0000;
1585  let Inst{5}     = i1;
1586  let Inst{4-0}   = Zdn;
1587
1588  let Constraints = "$Zdn = $_Zdn";
1589  let DestructiveInstType = DestructiveOther;
1590  let ElementSize = zprty.ElementSize;
1591}
1592
1593multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, Operand imm_ty> {
1594  def _H : sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>;
1595  def _S : sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>;
1596  def _D : sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>;
1597}
1598
1599class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm,
1600                       ZPRRegOp zprty>
1601: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
1602  asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",
1603  "",
1604  []>, Sched<[]> {
1605  bits<3> Pg;
1606  bits<5> Zdn;
1607  bits<5> Zm;
1608  let Inst{31-24} = 0b01100101;
1609  let Inst{23-22} = sz;
1610  let Inst{21-20} = 0b00;
1611  let Inst{19-16} = opc;
1612  let Inst{15-13} = 0b100;
1613  let Inst{12-10} = Pg;
1614  let Inst{9-5}   = Zm;
1615  let Inst{4-0}   = Zdn;
1616
1617  let Constraints = "$Zdn = $_Zdn";
1618  let DestructiveInstType = DestructiveOther;
1619  let ElementSize = zprty.ElementSize;
1620}
1621
1622multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps,
1623                            SDPatternOperator op, DestructiveInstTypeEnum flags,
1624                            string revname="", bit isReverseInstr=0> {
1625  let DestructiveInstType = flags in {
1626  def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>,
1627           SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
1628  def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>,
1629           SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
1630  def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>,
1631           SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
1632  }
1633
1634  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
1635  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
1636  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
1637}
1638
1639multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm,
1640                                   SDPatternOperator op> {
1641  def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>;
1642  def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>;
1643  def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>;
1644
1645  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1646  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1647  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1648}
1649
1650multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> {
1651  def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;
1652  def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
1653  def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;
1654
1655  def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>;
1656  def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>;
1657  def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>;
1658}
1659
1660class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty>
1661: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, imm32_0_7:$imm3),
1662  asm, "\t$Zdn, $_Zdn, $Zm, $imm3",
1663  "",
1664  []>, Sched<[]> {
1665  bits<5> Zdn;
1666  bits<5> Zm;
1667  bits<3> imm3;
1668  let Inst{31-24} = 0b01100101;
1669  let Inst{23-22} = sz;
1670  let Inst{21-19} = 0b010;
1671  let Inst{18-16} = imm3;
1672  let Inst{15-10} = 0b100000;
1673  let Inst{9-5}   = Zm;
1674  let Inst{4-0}   = Zdn;
1675
1676  let Constraints = "$Zdn = $_Zdn";
1677  let DestructiveInstType = DestructiveOther;
1678  let ElementSize = ElementSizeNone;
1679}
1680
1681multiclass sve_fp_ftmad<string asm, SDPatternOperator op> {
1682  def _H : sve_fp_ftmad<0b01, asm, ZPR16>;
1683  def _S : sve_fp_ftmad<0b10, asm, ZPR32>;
1684  def _D : sve_fp_ftmad<0b11, asm, ZPR64>;
1685
1686  def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 imm32_0_7:$imm))),
1687            (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, imm32_0_7:$imm)>;
1688  def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 imm32_0_7:$imm))),
1689            (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, imm32_0_7:$imm)>;
1690  def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 imm32_0_7:$imm))),
1691            (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, imm32_0_7:$imm)>;
1692}
1693
1694//===----------------------------------------------------------------------===//
1695// SVE Floating Point Arithmetic - Unpredicated Group
1696//===----------------------------------------------------------------------===//
1697
1698class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty>
1699: I<(outs zprty:$Zd), (ins  zprty:$Zn, zprty:$Zm),
1700  asm, "\t$Zd, $Zn, $Zm",
1701  "",
1702  []>, Sched<[]> {
1703  bits<5> Zd;
1704  bits<5> Zm;
1705  bits<5> Zn;
1706  let Inst{31-24} = 0b01100101;
1707  let Inst{23-22} = sz;
1708  let Inst{21}    = 0b0;
1709  let Inst{20-16} = Zm;
1710  let Inst{15-13} = 0b000;
1711  let Inst{12-10} = opc;
1712  let Inst{9-5}   = Zn;
1713  let Inst{4-0}   = Zd;
1714}
1715
1716multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op,
1717                           SDPatternOperator predicated_op = null_frag> {
1718  def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;
1719  def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;
1720  def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;
1721
1722  def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
1723  def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
1724  def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
1725
1726  def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
1727  def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
1728  def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
1729}
1730
1731multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> {
1732  def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>;
1733  def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>;
1734  def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>;
1735
1736  def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
1737  def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
1738  def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
1739}
1740
1741//===----------------------------------------------------------------------===//
1742// SVE Floating Point Fused Multiply-Add Group
1743//===----------------------------------------------------------------------===//
1744
1745class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty>
1746: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),
1747  asm, "\t$Zda, $Pg/m, $Zn, $Zm",
1748  "",
1749  []>, Sched<[]> {
1750  bits<3> Pg;
1751  bits<5> Zda;
1752  bits<5> Zm;
1753  bits<5> Zn;
1754  let Inst{31-24} = 0b01100101;
1755  let Inst{23-22} = sz;
1756  let Inst{21}    = 0b1;
1757  let Inst{20-16} = Zm;
1758  let Inst{15}    = 0b0;
1759  let Inst{14-13} = opc;
1760  let Inst{12-10} = Pg;
1761  let Inst{9-5}   = Zn;
1762  let Inst{4-0}   = Zda;
1763
1764  let Constraints = "$Zda = $_Zda";
1765  let DestructiveInstType = DestructiveOther;
1766  let ElementSize = zprty.ElementSize;
1767}
1768
1769multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, SDPatternOperator op> {
1770  def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>;
1771  def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>;
1772  def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>;
1773
1774  def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
1775  def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
1776  def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
1777}
1778
1779class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm,
1780                         ZPRRegOp zprty>
1781: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za),
1782  asm, "\t$Zdn, $Pg/m, $Zm, $Za",
1783  "",
1784  []>, Sched<[]> {
1785  bits<3> Pg;
1786  bits<5> Za;
1787  bits<5> Zdn;
1788  bits<5> Zm;
1789  let Inst{31-24} = 0b01100101;
1790  let Inst{23-22} = sz;
1791  let Inst{21}    = 0b1;
1792  let Inst{20-16} = Za;
1793  let Inst{15}    = 0b1;
1794  let Inst{14-13} = opc;
1795  let Inst{12-10} = Pg;
1796  let Inst{9-5}   = Zm;
1797  let Inst{4-0}   = Zdn;
1798
1799  let Constraints = "$Zdn = $_Zdn";
1800  let DestructiveInstType = DestructiveOther;
1801  let ElementSize = zprty.ElementSize;
1802}
1803
1804multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op> {
1805  def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>;
1806  def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>;
1807  def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>;
1808
1809  def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
1810  def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
1811  def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
1812}
1813
1814//===----------------------------------------------------------------------===//
1815// SVE Floating Point Multiply-Add - Indexed Group
1816//===----------------------------------------------------------------------===//
1817
1818class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm,
1819                                 ZPRRegOp zprty1,
1820                                 ZPRRegOp zprty2, Operand itype>
1821: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop),
1822  asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {
1823  bits<5> Zda;
1824  bits<5> Zn;
1825  let Inst{31-24} = 0b01100100;
1826  let Inst{23-22} = sz;
1827  let Inst{21}    = 0b1;
1828  let Inst{15-11} = 0;
1829  let Inst{10}    = opc;
1830  let Inst{9-5}   = Zn;
1831  let Inst{4-0}   = Zda;
1832
1833  let Constraints = "$Zda = $_Zda";
1834  let DestructiveInstType = DestructiveOther;
1835  let ElementSize = ElementSizeNone;
1836}
1837
1838multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm,
1839                                      SDPatternOperator op> {
1840  def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> {
1841    bits<3> Zm;
1842    bits<3> iop;
1843    let Inst{22} = iop{2};
1844    let Inst{20-19} = iop{1-0};
1845    let Inst{18-16} = Zm;
1846  }
1847  def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> {
1848    bits<3> Zm;
1849    bits<2> iop;
1850    let Inst{20-19} = iop;
1851    let Inst{18-16} = Zm;
1852  }
1853  def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> {
1854    bits<4> Zm;
1855    bit iop;
1856    let Inst{20} = iop;
1857    let Inst{19-16} = Zm;
1858  }
1859
1860  def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))),
1861            (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>;
1862  def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))),
1863            (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>;
1864  def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))),
1865            (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>;
1866}
1867
1868
1869//===----------------------------------------------------------------------===//
1870// SVE Floating Point Multiply - Indexed Group
1871//===----------------------------------------------------------------------===//
1872
1873class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty,
1874                                      ZPRRegOp zprty2, Operand itype>
1875: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop),
1876  asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> {
1877  bits<5> Zd;
1878  bits<5> Zn;
1879  let Inst{31-24} = 0b01100100;
1880  let Inst{23-22} = sz;
1881  let Inst{21}    = 0b1;
1882  let Inst{15-10} = 0b001000;
1883  let Inst{9-5}   = Zn;
1884  let Inst{4-0}   = Zd;
1885}
1886
1887multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> {
1888  def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> {
1889    bits<3> Zm;
1890    bits<3> iop;
1891    let Inst{22} = iop{2};
1892    let Inst{20-19} = iop{1-0};
1893    let Inst{18-16} = Zm;
1894  }
1895  def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> {
1896    bits<3> Zm;
1897    bits<2> iop;
1898    let Inst{20-19} = iop;
1899    let Inst{18-16} = Zm;
1900  }
1901  def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> {
1902    bits<4> Zm;
1903    bit iop;
1904    let Inst{20} = iop;
1905    let Inst{19-16} = Zm;
1906  }
1907
1908  def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))),
1909            (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>;
1910  def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))),
1911            (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>;
1912  def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))),
1913            (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>;
1914}
1915
1916//===----------------------------------------------------------------------===//
1917// SVE Floating Point Complex Multiply-Add Group
1918//===----------------------------------------------------------------------===//
1919
1920class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty>
1921: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm,
1922                        complexrotateop:$imm),
1923  asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm",
1924  "", []>, Sched<[]> {
1925  bits<5> Zda;
1926  bits<3> Pg;
1927  bits<5> Zn;
1928  bits<5> Zm;
1929  bits<2> imm;
1930  let Inst{31-24} = 0b01100100;
1931  let Inst{23-22} = sz;
1932  let Inst{21}    = 0;
1933  let Inst{20-16} = Zm;
1934  let Inst{15}    = 0;
1935  let Inst{14-13} = imm;
1936  let Inst{12-10} = Pg;
1937  let Inst{9-5}   = Zn;
1938  let Inst{4-0}   = Zda;
1939
1940  let Constraints = "$Zda = $_Zda";
1941  let DestructiveInstType = DestructiveOther;
1942  let ElementSize = zprty.ElementSize;
1943}
1944
1945multiclass sve_fp_fcmla<string asm, SDPatternOperator op> {
1946  def _H : sve_fp_fcmla<0b01, asm, ZPR16>;
1947  def _S : sve_fp_fcmla<0b10, asm, ZPR32>;
1948  def _D : sve_fp_fcmla<0b11, asm, ZPR64>;
1949
1950  def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))),
1951            (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>;
1952  def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))),
1953            (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>;
1954  def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))),
1955            (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>;
1956}
1957
1958//===----------------------------------------------------------------------===//
1959// SVE Floating Point Complex Multiply-Add - Indexed Group
1960//===----------------------------------------------------------------------===//
1961
1962class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm,
1963                                   ZPRRegOp zprty,
1964                                   ZPRRegOp zprty2, Operand itype>
1965: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop,
1966                        complexrotateop:$imm),
1967  asm, "\t$Zda, $Zn, $Zm$iop, $imm",
1968  "", []>, Sched<[]> {
1969  bits<5> Zda;
1970  bits<5> Zn;
1971  bits<2> imm;
1972  let Inst{31-24} = 0b01100100;
1973  let Inst{23-22} = sz;
1974  let Inst{21}    = 0b1;
1975  let Inst{15-12} = 0b0001;
1976  let Inst{11-10} = imm;
1977  let Inst{9-5}   = Zn;
1978  let Inst{4-0}   = Zda;
1979
1980  let Constraints = "$Zda = $_Zda";
1981  let DestructiveInstType = DestructiveOther;
1982  let ElementSize = ElementSizeNone;
1983}
1984
1985multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> {
1986  def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> {
1987    bits<3> Zm;
1988    bits<2> iop;
1989    let Inst{20-19} = iop;
1990    let Inst{18-16} = Zm;
1991  }
1992  def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> {
1993    bits<4> Zm;
1994    bits<1> iop;
1995    let Inst{20} = iop;
1996    let Inst{19-16} = Zm;
1997  }
1998
1999  def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))),
2000            (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>;
2001  def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))),
2002            (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>;
2003}
2004
2005//===----------------------------------------------------------------------===//
2006// SVE Floating Point Complex Addition Group
2007//===----------------------------------------------------------------------===//
2008
2009class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty>
2010: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm,
2011                        complexrotateopodd:$imm),
2012  asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm",
2013  "",
2014  []>, Sched<[]> {
2015  bits<5> Zdn;
2016  bits<5> Zm;
2017  bits<3> Pg;
2018  bit imm;
2019  let Inst{31-24} = 0b01100100;
2020  let Inst{23-22} = sz;
2021  let Inst{21-17} = 0;
2022  let Inst{16}    = imm;
2023  let Inst{15-13} = 0b100;
2024  let Inst{12-10} = Pg;
2025  let Inst{9-5}   = Zm;
2026  let Inst{4-0}   = Zdn;
2027
2028  let Constraints = "$Zdn = $_Zdn";
2029  let DestructiveInstType = DestructiveOther;
2030  let ElementSize = zprty.ElementSize;
2031}
2032
2033multiclass sve_fp_fcadd<string asm, SDPatternOperator op> {
2034  def _H : sve_fp_fcadd<0b01, asm, ZPR16>;
2035  def _S : sve_fp_fcadd<0b10, asm, ZPR32>;
2036  def _D : sve_fp_fcadd<0b11, asm, ZPR64>;
2037
2038  def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))),
2039            (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>;
2040  def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))),
2041            (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>;
2042  def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))),
2043            (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>;
2044}
2045
2046//===----------------------------------------------------------------------===//
2047// SVE2 Floating Point Convert Group
2048//===----------------------------------------------------------------------===//
2049
2050class sve2_fp_convert_precision<bits<4> opc, string asm,
2051                                ZPRRegOp zprty1, ZPRRegOp zprty2>
2052: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn),
2053  asm, "\t$Zd, $Pg/m, $Zn",
2054  "",
2055  []>, Sched<[]> {
2056  bits<5> Zd;
2057  bits<5> Zn;
2058  bits<3> Pg;
2059  let Inst{31-24} = 0b01100100;
2060  let Inst{23-22} = opc{3-2};
2061  let Inst{21-18} = 0b0010;
2062  let Inst{17-16} = opc{1-0};
2063  let Inst{15-13} = 0b101;
2064  let Inst{12-10} = Pg;
2065  let Inst{9-5}   = Zn;
2066  let Inst{4-0}   = Zd;
2067
2068  let Constraints = "$Zd = $_Zd";
2069}
2070
2071multiclass sve2_fp_convert_down_narrow<string asm, string op> {
2072  def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>;
2073  def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>;
2074
2075  def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>;
2076  def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
2077}
2078
2079multiclass sve2_fp_convert_up_long<string asm, string op> {
2080  def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>;
2081  def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>;
2082
2083  def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>;
2084  def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>;
2085}
2086
2087multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> {
2088  def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>;
2089
2090  def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
2091}
2092
2093//===----------------------------------------------------------------------===//
2094// SVE2 Floating Point Pairwise Group
2095//===----------------------------------------------------------------------===//
2096
2097class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm,
2098                            ZPRRegOp zprty>
2099: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
2100  asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",
2101  "",
2102  []>, Sched<[]> {
2103  bits<3> Pg;
2104  bits<5> Zm;
2105  bits<5> Zdn;
2106  let Inst{31-24} = 0b01100100;
2107  let Inst{23-22} = sz;
2108  let Inst{21-19} = 0b010;
2109  let Inst{18-16} = opc;
2110  let Inst{15-13} = 0b100;
2111  let Inst{12-10} = Pg;
2112  let Inst{9-5}   = Zm;
2113  let Inst{4-0}   = Zdn;
2114
2115  let Constraints = "$Zdn = $_Zdn";
2116  let DestructiveInstType = DestructiveOther;
2117  let ElementSize = zprty.ElementSize;
2118}
2119
2120multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm,
2121                                 SDPatternOperator op> {
2122  def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>;
2123  def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>;
2124  def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>;
2125
2126  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
2127  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
2128  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
2129}
2130
2131//===----------------------------------------------------------------------===//
2132// SVE2 Floating Point Widening Multiply-Add - Indexed Group
2133//===----------------------------------------------------------------------===//
2134
2135class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm>
2136: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm,
2137                        VectorIndexH32b:$iop),
2138  asm, "\t$Zda, $Zn, $Zm$iop",
2139  "",
2140  []>, Sched<[]> {
2141  bits<5> Zda;
2142  bits<5> Zn;
2143  bits<3> Zm;
2144  bits<3> iop;
2145  let Inst{31-21} = 0b01100100101;
2146  let Inst{20-19} = iop{2-1};
2147  let Inst{18-16} = Zm;
2148  let Inst{15-14} = 0b01;
2149  let Inst{13}    = opc{1};
2150  let Inst{12}    = 0b0;
2151  let Inst{11}    = iop{0};
2152  let Inst{10}    = opc{0};
2153  let Inst{9-5}   = Zn;
2154  let Inst{4-0}   = Zda;
2155
2156  let Constraints = "$Zda = $_Zda";
2157  let DestructiveInstType = DestructiveOther;
2158  let ElementSize = ElementSizeNone;
2159}
2160
2161multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm,
2162                                            SDPatternOperator op> {
2163  def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>;
2164  def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>;
2165}
2166
2167//===----------------------------------------------------------------------===//
2168// SVE2 Floating Point Widening Multiply-Add Group
2169//===----------------------------------------------------------------------===//
2170
2171class sve2_fp_mla_long<bits<2> opc, string asm>
2172: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
2173  asm, "\t$Zda, $Zn, $Zm",
2174  "",
2175  []>, Sched<[]> {
2176  bits<5> Zda;
2177  bits<5> Zn;
2178  bits<5> Zm;
2179  let Inst{31-21} = 0b01100100101;
2180  let Inst{20-16} = Zm;
2181  let Inst{15-14} = 0b10;
2182  let Inst{13}    = opc{1};
2183  let Inst{12-11} = 0b00;
2184  let Inst{10}    = opc{0};
2185  let Inst{9-5}   = Zn;
2186  let Inst{4-0}   = Zda;
2187
2188  let Constraints = "$Zda = $_Zda";
2189  let DestructiveInstType = DestructiveOther;
2190  let ElementSize = ElementSizeNone;
2191}
2192
2193multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> {
2194  def NAME : sve2_fp_mla_long<opc, asm>;
2195  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>;
2196}
2197
2198//===----------------------------------------------------------------------===//
2199// SVE Stack Allocation Group
2200//===----------------------------------------------------------------------===//
2201
2202class sve_int_arith_vl<bit opc, string asm>
2203: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6),
2204  asm, "\t$Rd, $Rn, $imm6",
2205  "",
2206  []>, Sched<[]> {
2207  bits<5> Rd;
2208  bits<5> Rn;
2209  bits<6> imm6;
2210  let Inst{31-23} = 0b000001000;
2211  let Inst{22}    = opc;
2212  let Inst{21}    = 0b1;
2213  let Inst{20-16} = Rn;
2214  let Inst{15-11} = 0b01010;
2215  let Inst{10-5}  = imm6;
2216  let Inst{4-0}   = Rd;
2217}
2218
2219class sve_int_read_vl_a<bit op, bits<5> opc2, string asm>
2220: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6),
2221  asm, "\t$Rd, $imm6",
2222  "",
2223  []>, Sched<[]> {
2224  bits<5> Rd;
2225  bits<6> imm6;
2226  let Inst{31-23} = 0b000001001;
2227  let Inst{22}    = op;
2228  let Inst{21}    = 0b1;
2229  let Inst{20-16} = opc2{4-0};
2230  let Inst{15-11} = 0b01010;
2231  let Inst{10-5}  = imm6;
2232  let Inst{4-0}   = Rd;
2233}
2234
2235//===----------------------------------------------------------------------===//
2236// SVE Permute - In Lane Group
2237//===----------------------------------------------------------------------===//
2238
2239class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm,
2240                               ZPRRegOp zprty>
2241: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
2242  asm, "\t$Zd, $Zn, $Zm",
2243  "",
2244  []>, Sched<[]> {
2245  bits<5> Zd;
2246  bits<5> Zm;
2247  bits<5> Zn;
2248  let Inst{31-24} = 0b00000101;
2249  let Inst{23-22} = sz8_64;
2250  let Inst{21}    = 0b1;
2251  let Inst{20-16} = Zm;
2252  let Inst{15-13} = 0b011;
2253  let Inst{12-10} = opc;
2254  let Inst{9-5}   = Zn;
2255  let Inst{4-0}   = Zd;
2256}
2257
2258multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm,
2259                                    SDPatternOperator op> {
2260  def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>;
2261  def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>;
2262  def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>;
2263  def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>;
2264
2265  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2266  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2267  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2268  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2269
2270  def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
2271  def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>;
2272  def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
2273  def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>;
2274  def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>;
2275  def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
2276
2277  def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
2278}
2279
2280//===----------------------------------------------------------------------===//
2281// SVE Floating Point Unary Operations Group
2282//===----------------------------------------------------------------------===//
2283
2284class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype,
2285                      RegisterOperand o_zprtype, ElementSizeEnum Sz>
2286: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn),
2287  asm, "\t$Zd, $Pg/m, $Zn",
2288  "",
2289  []>, Sched<[]> {
2290  bits<3> Pg;
2291  bits<5> Zd;
2292  bits<5> Zn;
2293  let Inst{31-24} = 0b01100101;
2294  let Inst{23-22} = opc{6-5};
2295  let Inst{21}    = 0b0;
2296  let Inst{20-16} = opc{4-0};
2297  let Inst{15-13} = 0b101;
2298  let Inst{12-10} = Pg;
2299  let Inst{9-5}   = Zn;
2300  let Inst{4-0}   = Zd;
2301
2302  let Constraints = "$Zd = $_Zd";
2303  let DestructiveInstType = DestructiveOther;
2304  let ElementSize = Sz;
2305}
2306
2307multiclass sve_fp_2op_p_zd<bits<7> opc, string asm,
2308                           RegisterOperand i_zprtype,
2309                           RegisterOperand o_zprtype,
2310                           SDPatternOperator int_op,
2311                           SDPatternOperator ir_op, ValueType vt1,
2312                           ValueType vt2, ValueType vt3, ElementSizeEnum Sz> {
2313  def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>;
2314
2315  // convert vt1 to a packed type for the intrinsic patterns
2316  defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16,
2317                           !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16,
2318                           !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32,
2319                           1 : vt1);
2320
2321  // convert vt3 to a packed type for the intrinsic patterns
2322  defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16,
2323                           !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16,
2324                           !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32,
2325                           1 : vt3);
2326
2327  def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>;
2328
2329  def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>;
2330}
2331
2332multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm,
2333                            RegisterOperand i_zprtype,
2334                            RegisterOperand o_zprtype,
2335                            SDPatternOperator int_op,
2336                            SDPatternOperator ir_op, ValueType vt1,
2337                            ValueType vt2, ValueType vt3, ElementSizeEnum Sz> {
2338  def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>;
2339
2340  // convert vt1 to a packed type for the intrinsic patterns
2341  defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16,
2342                           !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16,
2343                           !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32,
2344                           1 : vt1);
2345
2346  def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>;
2347
2348  def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>;
2349}
2350
2351multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> {
2352  def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>;
2353  def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>;
2354  def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>;
2355
2356  def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
2357  def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
2358  def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
2359  def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
2360  def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
2361  def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
2362}
2363
2364multiclass sve2_fp_flogb<string asm, SDPatternOperator op> {
2365  def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>;
2366  def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>;
2367  def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>;
2368
2369  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
2370  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
2371  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
2372}
2373
2374multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> {
2375  def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>;
2376  def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>;
2377}
2378
2379//===----------------------------------------------------------------------===//
2380// SVE Floating Point Unary Operations - Unpredicated Group
2381//===----------------------------------------------------------------------===//
2382
2383class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm,
2384                      ZPRRegOp zprty>
2385: I<(outs zprty:$Zd), (ins zprty:$Zn),
2386  asm, "\t$Zd, $Zn",
2387  "",
2388  []>, Sched<[]> {
2389  bits<5> Zd;
2390  bits<5> Zn;
2391  let Inst{31-24} = 0b01100101;
2392  let Inst{23-22} = sz;
2393  let Inst{21-19} = 0b001;
2394  let Inst{18-16} = opc;
2395  let Inst{15-10} = 0b001100;
2396  let Inst{9-5}   = Zn;
2397  let Inst{4-0}   = Zd;
2398}
2399
2400multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> {
2401  def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>;
2402  def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>;
2403  def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>;
2404
2405  def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>;
2406  def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>;
2407  def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>;
2408}
2409
2410//===----------------------------------------------------------------------===//
2411// SVE Integer Arithmetic - Binary Predicated Group
2412//===----------------------------------------------------------------------===//
2413
2414class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc,
2415                                string asm, ZPRRegOp zprty>
2416: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
2417  asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> {
2418  bits<3> Pg;
2419  bits<5> Zdn;
2420  bits<5> Zm;
2421  let Inst{31-24} = 0b00000100;
2422  let Inst{23-22} = sz8_64;
2423  let Inst{21}    = 0b0;
2424  let Inst{20-19} = fmt;
2425  let Inst{18-16} = opc;
2426  let Inst{15-13} = 0b000;
2427  let Inst{12-10} = Pg;
2428  let Inst{9-5}   = Zm;
2429  let Inst{4-0}   = Zdn;
2430
2431  let Constraints = "$Zdn = $_Zdn";
2432  let DestructiveInstType = DestructiveOther;
2433  let ElementSize = zprty.ElementSize;
2434}
2435
2436multiclass sve_int_bin_pred_log<bits<3> opc, string asm, SDPatternOperator op> {
2437  def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>;
2438  def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>;
2439  def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>;
2440  def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>;
2441
2442  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2443  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2444  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2445  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2446}
2447
2448multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps,
2449                                   SDPatternOperator op,
2450                                   DestructiveInstTypeEnum flags,
2451                                   string revname="", bit isReverseInstr=0> {
2452  let DestructiveInstType = flags in {
2453  def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>,
2454             SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;
2455  def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>,
2456             SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
2457  def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>,
2458             SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
2459  def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>,
2460             SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
2461  }
2462
2463  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2464  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2465  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2466  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2467}
2468
2469multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps,
2470                                   SDPatternOperator op,
2471                                   DestructiveInstTypeEnum flags> {
2472  let DestructiveInstType = flags in {
2473  def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>,
2474             SVEPseudo2Instr<Ps # _B, 1>;
2475  def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>,
2476             SVEPseudo2Instr<Ps # _H, 1>;
2477  def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>,
2478             SVEPseudo2Instr<Ps # _S, 1>;
2479  def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>,
2480             SVEPseudo2Instr<Ps # _D, 1>;
2481  }
2482
2483  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2484  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2485  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2486  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2487}
2488
2489multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps,
2490                                   SDPatternOperator op,
2491                                   DestructiveInstTypeEnum flags> {
2492  let DestructiveInstType = flags in {
2493  def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>,
2494             SVEPseudo2Instr<Ps # _B, 1>;
2495  def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>,
2496             SVEPseudo2Instr<Ps # _H, 1>;
2497  def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>,
2498             SVEPseudo2Instr<Ps # _S, 1>;
2499  def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>,
2500             SVEPseudo2Instr<Ps # _D, 1>;
2501  }
2502
2503  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2504  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2505  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2506  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2507}
2508
2509// Special case for divides which are not defined for 8b/16b elements.
2510multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps,
2511                                       SDPatternOperator op,
2512                                       DestructiveInstTypeEnum flags,
2513                                       string revname="", bit isReverseInstr=0> {
2514  let DestructiveInstType = flags in {
2515  def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>,
2516             SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
2517  def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>,
2518             SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
2519  }
2520
2521  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2522  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2523}
2524
2525//===----------------------------------------------------------------------===//
2526// SVE Integer Multiply-Add Group
2527//===----------------------------------------------------------------------===//
2528
2529class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm,
2530                                ZPRRegOp zprty>
2531: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za),
2532  asm, "\t$Zdn, $Pg/m, $Zm, $Za",
2533  "",
2534  []>, Sched<[]> {
2535  bits<3> Pg;
2536  bits<5> Zdn;
2537  bits<5> Za;
2538  bits<5> Zm;
2539  let Inst{31-24} = 0b00000100;
2540  let Inst{23-22} = sz8_64;
2541  let Inst{21}    = 0b0;
2542  let Inst{20-16} = Zm;
2543  let Inst{15-14} = 0b11;
2544  let Inst{13}    = opc;
2545  let Inst{12-10} = Pg;
2546  let Inst{9-5}   = Za;
2547  let Inst{4-0}   = Zdn;
2548
2549  let Constraints = "$Zdn = $_Zdn";
2550  let DestructiveInstType = DestructiveOther;
2551  let ElementSize = zprty.ElementSize;
2552}
2553
2554multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> {
2555  def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>;
2556  def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>;
2557  def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>;
2558  def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>;
2559
2560  def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2561  def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2562  def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2563  def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2564}
2565
2566class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm,
2567                            ZPRRegOp zprty>
2568: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm),
2569  asm, "\t$Zda, $Pg/m, $Zn, $Zm",
2570  "",
2571  []>, Sched<[]> {
2572  bits<3> Pg;
2573  bits<5> Zda;
2574  bits<5> Zm;
2575  bits<5> Zn;
2576  let Inst{31-24} = 0b00000100;
2577  let Inst{23-22} = sz8_64;
2578  let Inst{21}    = 0b0;
2579  let Inst{20-16} = Zm;
2580  let Inst{15-14} = 0b01;
2581  let Inst{13}    = opc;
2582  let Inst{12-10} = Pg;
2583  let Inst{9-5}   = Zn;
2584  let Inst{4-0}   = Zda;
2585
2586  let Constraints = "$Zda = $_Zda";
2587  let DestructiveInstType = DestructiveOther;
2588  let ElementSize = zprty.ElementSize;
2589}
2590
2591multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op,
2592                                 SDPatternOperator outerop, SDPatternOperator mulop> {
2593  def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>;
2594  def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>;
2595  def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>;
2596  def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>;
2597
2598  def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2599  def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2600  def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2601  def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2602
2603  def : Pat<(outerop nxv16i8:$Op1, (mulop nxv16i1:$pred, nxv16i8:$Op2, nxv16i8:$Op3)),
2604            (!cast<Instruction>(NAME # _B) $pred, $Op1, $Op2, $Op3)>;
2605  def : Pat<(outerop nxv8i16:$Op1, (mulop nxv8i1:$pred, nxv8i16:$Op2, nxv8i16:$Op3)),
2606            (!cast<Instruction>(NAME # _H) $pred, $Op1, $Op2, $Op3)>;
2607  def : Pat<(outerop nxv4i32:$Op1, (mulop nxv4i1:$pred, nxv4i32:$Op2, nxv4i32:$Op3)),
2608            (!cast<Instruction>(NAME # _S) $pred, $Op1, $Op2, $Op3)>;
2609  def : Pat<(outerop nxv2i64:$Op1, (mulop nxv2i1:$pred, nxv2i64:$Op2, nxv2i64:$Op3)),
2610            (!cast<Instruction>(NAME # _D) $pred, $Op1, $Op2, $Op3)>;
2611}
2612
2613//===----------------------------------------------------------------------===//
2614// SVE2 Integer Multiply-Add - Unpredicated Group
2615//===----------------------------------------------------------------------===//
2616
2617class sve2_int_mla<bits<2> sz, bits<5> opc, string asm,
2618                   ZPRRegOp zprty1, ZPRRegOp zprty2>
2619: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm),
2620  asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
2621  bits<5> Zda;
2622  bits<5> Zn;
2623  bits<5> Zm;
2624  let Inst{31-24} = 0b01000100;
2625  let Inst{23-22} = sz;
2626  let Inst{21}    = 0b0;
2627  let Inst{20-16} = Zm;
2628  let Inst{15}    = 0b0;
2629  let Inst{14-10} = opc;
2630  let Inst{9-5}   = Zn;
2631  let Inst{4-0}   = Zda;
2632
2633  let Constraints = "$Zda = $_Zda";
2634  let DestructiveInstType = DestructiveOther;
2635  let ElementSize = ElementSizeNone;
2636}
2637
2638multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> {
2639  def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>;
2640  def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>;
2641  def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>;
2642  def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>;
2643
2644  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2645  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2646  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2647  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2648}
2649
2650multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> {
2651  def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>;
2652  def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>;
2653  def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>;
2654
2655  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;
2656  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;
2657  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;
2658}
2659
2660//===----------------------------------------------------------------------===//
2661// SVE2 Integer Multiply-Add - Indexed Group
2662//===----------------------------------------------------------------------===//
2663
2664class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm,
2665                                   ZPRRegOp zprty1, ZPRRegOp zprty2,
2666                                   ZPRRegOp zprty3, Operand itype>
2667: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop),
2668  asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> {
2669  bits<5> Zda;
2670  bits<5> Zn;
2671  let Inst{31-24} = 0b01000100;
2672  let Inst{23-22} = sz;
2673  let Inst{21}    = 0b1;
2674  let Inst{15-10} = opc;
2675  let Inst{9-5}   = Zn;
2676  let Inst{4-0}   = Zda;
2677
2678  let Constraints = "$Zda = $_Zda";
2679  let DestructiveInstType = DestructiveOther;
2680  let ElementSize = ElementSizeNone;
2681}
2682
2683multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm,
2684                                        SDPatternOperator op> {
2685  def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {
2686    bits<3> Zm;
2687    bits<3> iop;
2688    let Inst{22} = iop{2};
2689    let Inst{20-19} = iop{1-0};
2690    let Inst{18-16} = Zm;
2691  }
2692  def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {
2693    bits<3> Zm;
2694    bits<2> iop;
2695    let Inst{20-19} = iop;
2696    let Inst{18-16} = Zm;
2697  }
2698  def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {
2699    bits<4> Zm;
2700    bit iop;
2701    let Inst{20} = iop;
2702    let Inst{19-16} = Zm;
2703  }
2704
2705  def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
2706  def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
2707  def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
2708}
2709
2710//===----------------------------------------------------------------------===//
2711// SVE2 Integer Multiply-Add Long - Indexed Group
2712//===----------------------------------------------------------------------===//
2713
2714multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm,
2715                                             SDPatternOperator op> {
2716  def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} },
2717                                        asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> {
2718    bits<3> Zm;
2719    bits<3> iop;
2720    let Inst{20-19} = iop{2-1};
2721    let Inst{18-16} = Zm;
2722    let Inst{11} = iop{0};
2723  }
2724  def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} },
2725                                        asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> {
2726    bits<4> Zm;
2727    bits<2> iop;
2728    let Inst{20} = iop{1};
2729    let Inst{19-16} = Zm;
2730    let Inst{11} = iop{0};
2731  }
2732
2733  def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>;
2734  def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>;
2735}
2736
2737//===----------------------------------------------------------------------===//
2738// SVE Integer Dot Product Group
2739//===----------------------------------------------------------------------===//
2740
2741class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1,
2742                   ZPRRegOp zprty2>
2743: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm,
2744  "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
2745  bits<5> Zda;
2746  bits<5> Zn;
2747  bits<5> Zm;
2748  let Inst{31-23} = 0b010001001;
2749  let Inst{22}    = sz;
2750  let Inst{21}    = 0;
2751  let Inst{20-16} = Zm;
2752  let Inst{15-11} = 0;
2753  let Inst{10}    = U;
2754  let Inst{9-5}   = Zn;
2755  let Inst{4-0}   = Zda;
2756
2757  let Constraints = "$Zda = $_Zda";
2758  let DestructiveInstType = DestructiveOther;
2759}
2760
2761multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> {
2762  def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>;
2763  def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>;
2764
2765  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32,  nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>;
2766  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>;
2767}
2768
2769//===----------------------------------------------------------------------===//
2770// SVE Integer Dot Product Group - Indexed Group
2771//===----------------------------------------------------------------------===//
2772
2773class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm,
2774                                   ZPRRegOp zprty1, ZPRRegOp zprty2,
2775                                   ZPRRegOp zprty3, Operand itype>
2776: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop),
2777  asm, "\t$Zda, $Zn, $Zm$iop",
2778  "", []>, Sched<[]> {
2779  bits<5> Zda;
2780  bits<5> Zn;
2781  let Inst{31-23} = 0b010001001;
2782  let Inst{22}    = sz;
2783  let Inst{21}    = 0b1;
2784  let Inst{15-11} = 0;
2785  let Inst{10}    = U;
2786  let Inst{9-5}   = Zn;
2787  let Inst{4-0}   = Zda;
2788
2789  let Constraints = "$Zda = $_Zda";
2790  let DestructiveInstType = DestructiveOther;
2791}
2792
2793multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm,
2794                                        SDPatternOperator op> {
2795  def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> {
2796    bits<2> iop;
2797    bits<3> Zm;
2798    let Inst{20-19} = iop;
2799    let Inst{18-16} = Zm;
2800  }
2801  def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> {
2802    bits<1> iop;
2803    bits<4> Zm;
2804    let Inst{20} = iop;
2805    let Inst{19-16} = Zm;
2806  }
2807
2808  def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv16i8:$Op2, nxv16i8:$Op3, (i32 VectorIndexS32b_timm:$idx))),
2809            (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>;
2810  def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv8i16:$Op2, nxv8i16:$Op3, (i32 VectorIndexD32b_timm:$idx))),
2811            (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>;
2812}
2813
2814//===----------------------------------------------------------------------===//
2815// SVE2 Complex Integer Dot Product Group
2816//===----------------------------------------------------------------------===//
2817
2818class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm,
2819                             ZPRRegOp zprty1, ZPRRegOp zprty2>
2820: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm,
2821                         complexrotateop:$rot),
2822  asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> {
2823  bits<5> Zda;
2824  bits<5> Zn;
2825  bits<5> Zm;
2826  bits<2> rot;
2827  let Inst{31-24} = 0b01000100;
2828  let Inst{23-22} = sz;
2829  let Inst{21}    = 0b0;
2830  let Inst{20-16} = Zm;
2831  let Inst{15-12} = opc;
2832  let Inst{11-10} = rot;
2833  let Inst{9-5}   = Zn;
2834  let Inst{4-0}   = Zda;
2835
2836  let Constraints = "$Zda = $_Zda";
2837  let DestructiveInstType = DestructiveOther;
2838  let ElementSize = ElementSizeNone;
2839}
2840
2841multiclass sve2_cintx_dot<string asm, SDPatternOperator op> {
2842  def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>;
2843  def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>;
2844
2845  def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3),
2846                         (i32 complexrotateop:$imm))),
2847            (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>;
2848  def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3),
2849                         (i32 complexrotateop:$imm))),
2850            (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>;
2851}
2852
2853//===----------------------------------------------------------------------===//
2854// SVE2 Complex Multiply-Add Group
2855//===----------------------------------------------------------------------===//
2856
2857multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> {
2858  def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>;
2859  def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>;
2860  def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>;
2861  def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>;
2862
2863  def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>;
2864  def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>;
2865  def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>;
2866  def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>;
2867}
2868
2869//===----------------------------------------------------------------------===//
2870// SVE2 Complex Integer Dot Product - Indexed Group
2871//===----------------------------------------------------------------------===//
2872
2873class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm,
2874                                     ZPRRegOp zprty1, ZPRRegOp zprty2,
2875                                     ZPRRegOp zprty3, Operand itype>
2876: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop,
2877                         complexrotateop:$rot),
2878  asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> {
2879  bits<5> Zda;
2880  bits<5> Zn;
2881  bits<2> rot;
2882  let Inst{31-24} = 0b01000100;
2883  let Inst{23-22} = sz;
2884  let Inst{21}    = 0b1;
2885  let Inst{15-12} = opc;
2886  let Inst{11-10} = rot;
2887  let Inst{9-5}   = Zn;
2888  let Inst{4-0}   = Zda;
2889
2890  let Constraints = "$Zda = $_Zda";
2891  let DestructiveInstType = DestructiveOther;
2892  let ElementSize = ElementSizeNone;
2893}
2894
2895multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> {
2896  def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> {
2897    bits<2> iop;
2898    bits<3> Zm;
2899    let Inst{20-19} = iop;
2900    let Inst{18-16} = Zm;
2901  }
2902  def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> {
2903    bit iop;
2904    bits<4> Zm;
2905    let Inst{20} = iop;
2906    let Inst{19-16} = Zm;
2907  }
2908
2909  def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3),
2910                         (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))),
2911            (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>;
2912  def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3),
2913                         (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))),
2914            (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>;
2915}
2916
2917//===----------------------------------------------------------------------===//
2918// SVE2 Complex Multiply-Add - Indexed Group
2919//===----------------------------------------------------------------------===//
2920
2921multiclass sve2_cmla_by_indexed_elem<bit opc, string asm,
2922                                     SDPatternOperator op> {
2923  def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> {
2924    bits<2> iop;
2925    bits<3> Zm;
2926    let Inst{20-19} = iop;
2927    let Inst{18-16} = Zm;
2928  }
2929  def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> {
2930    bit iop;
2931    bits<4> Zm;
2932    let Inst{20} = iop;
2933    let Inst{19-16} = Zm;
2934  }
2935
2936  def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3),
2937                         (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))),
2938            (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>;
2939
2940  def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3),
2941                         (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))),
2942            (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>;
2943}
2944
2945//===----------------------------------------------------------------------===//
2946// SVE2 Integer Multiply - Unpredicated Group
2947//===----------------------------------------------------------------------===//
2948
2949class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty>
2950: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
2951  asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {
2952  bits<5> Zd;
2953  bits<5> Zm;
2954  bits<5> Zn;
2955  let Inst{31-24} = 0b00000100;
2956  let Inst{23-22} = sz;
2957  let Inst{21}    = 0b1;
2958  let Inst{20-16} = Zm;
2959  let Inst{15-13} = 0b011;
2960  let Inst{12-10} = opc;
2961  let Inst{9-5}   = Zn;
2962  let Inst{4-0}   = Zd;
2963}
2964
2965multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op,
2966                        SDPatternOperator op_pred = null_frag> {
2967  def _B : sve2_int_mul<0b00, opc, asm, ZPR8>;
2968  def _H : sve2_int_mul<0b01, opc, asm, ZPR16>;
2969  def _S : sve2_int_mul<0b10, opc, asm, ZPR32>;
2970  def _D : sve2_int_mul<0b11, opc, asm, ZPR64>;
2971
2972  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2973  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2974  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2975  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2976
2977  def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2978  def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
2979  def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
2980  def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
2981}
2982
2983multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> {
2984  def _B : sve2_int_mul<0b00, opc, asm, ZPR8>;
2985
2986  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
2987}
2988
2989//===----------------------------------------------------------------------===//
2990// SVE2 Integer Multiply - Indexed Group
2991//===----------------------------------------------------------------------===//
2992
2993class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm,
2994                                   ZPRRegOp zprty1, ZPRRegOp zprty2,
2995                                   ZPRRegOp zprty3, Operand itype>
2996: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop),
2997  asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> {
2998  bits<5> Zd;
2999  bits<5> Zn;
3000  let Inst{31-24} = 0b01000100;
3001  let Inst{23-22} = sz;
3002  let Inst{21}    = 0b1;
3003  let Inst{15-14} = 0b11;
3004  let Inst{13-10} = opc;
3005  let Inst{9-5}   = Zn;
3006  let Inst{4-0}   = Zd;
3007}
3008
3009multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm,
3010                                        SDPatternOperator op> {
3011  def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> {
3012    bits<3> Zm;
3013    bits<3> iop;
3014    let Inst{22} = iop{2};
3015    let Inst{20-19} = iop{1-0};
3016    let Inst{18-16} = Zm;
3017  }
3018  def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> {
3019    bits<3> Zm;
3020    bits<2> iop;
3021    let Inst{20-19} = iop;
3022    let Inst{18-16} = Zm;
3023  }
3024  def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> {
3025    bits<4> Zm;
3026    bit iop;
3027    let Inst{20} = iop;
3028    let Inst{19-16} = Zm;
3029  }
3030
3031  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>;
3032  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>;
3033  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>;
3034}
3035
3036multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm,
3037                                             SDPatternOperator op> {
3038  def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm,
3039                                        ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> {
3040    bits<3> Zm;
3041    bits<3> iop;
3042    let Inst{20-19} = iop{2-1};
3043    let Inst{18-16} = Zm;
3044    let Inst{11} = iop{0};
3045  }
3046  def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm,
3047                                        ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> {
3048    bits<4> Zm;
3049    bits<2> iop;
3050    let Inst{20} = iop{1};
3051    let Inst{19-16} = Zm;
3052    let Inst{11} = iop{0};
3053  }
3054
3055  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>;
3056  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>;
3057}
3058
3059//===----------------------------------------------------------------------===//
3060// SVE2 Integer - Predicated Group
3061//===----------------------------------------------------------------------===//
3062
3063class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm,
3064                          ZPRRegOp zprty>
3065: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
3066  asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> {
3067  bits<3> Pg;
3068  bits<5> Zm;
3069  bits<5> Zdn;
3070  let Inst{31-24} = 0b01000100;
3071  let Inst{23-22} = sz;
3072  let Inst{21-20} = 0b01;
3073  let Inst{20-16} = opc{5-1};
3074  let Inst{15-14} = 0b10;
3075  let Inst{13}    = opc{0};
3076  let Inst{12-10} = Pg;
3077  let Inst{9-5}   = Zm;
3078  let Inst{4-0}   = Zdn;
3079
3080  let Constraints = "$Zdn = $_Zdn";
3081  let DestructiveInstType = DestructiveOther;
3082  let ElementSize = zprty.ElementSize;
3083}
3084
3085multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op> {
3086  def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>;
3087  def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>;
3088  def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>;
3089  def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>;
3090
3091  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
3092  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
3093  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
3094  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
3095}
3096
3097class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm,
3098                                        ZPRRegOp zprty1, ZPRRegOp zprty2>
3099: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn),
3100  asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> {
3101  bits<3> Pg;
3102  bits<5> Zn;
3103  bits<5> Zda;
3104  let Inst{31-24} = 0b01000100;
3105  let Inst{23-22} = sz;
3106  let Inst{21-17} = 0b00010;
3107  let Inst{16}    = U;
3108  let Inst{15-13} = 0b101;
3109  let Inst{12-10} = Pg;
3110  let Inst{9-5}   = Zn;
3111  let Inst{4-0}   = Zda;
3112
3113  let Constraints = "$Zda = $_Zda";
3114  let DestructiveInstType = DestructiveOther;
3115  let ElementSize = zprty1.ElementSize;
3116}
3117
3118multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> {
3119  def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>;
3120  def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>;
3121  def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>;
3122
3123  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;
3124  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;
3125  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;
3126}
3127
3128class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc,
3129                            string asm, ZPRRegOp zprty>
3130: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
3131  asm, "\t$Zd, $Pg/m, $Zn",
3132  "",
3133  []>, Sched<[]> {
3134  bits<3> Pg;
3135  bits<5> Zd;
3136  bits<5> Zn;
3137  let Inst{31-24} = 0b01000100;
3138  let Inst{23-22} = sz;
3139  let Inst{21-20} = 0b00;
3140  let Inst{19}    = Q;
3141  let Inst{18}    = 0b0;
3142  let Inst{17-16} = opc;
3143  let Inst{15-13} = 0b101;
3144  let Inst{12-10} = Pg;
3145  let Inst{9-5}   = Zn;
3146  let Inst{4-0}   = Zd;
3147
3148  let Constraints = "$Zd = $_Zd";
3149  let DestructiveInstType = DestructiveOther;
3150  let ElementSize = zprty.ElementSize;
3151}
3152
3153multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm,
3154                                   SDPatternOperator op> {
3155  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>;
3156  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
3157}
3158
3159multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> {
3160  def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>;
3161  def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>;
3162  def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>;
3163  def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>;
3164
3165  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
3166  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
3167  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
3168  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
3169}
3170
3171//===----------------------------------------------------------------------===//
3172// SVE2 Widening Integer Arithmetic Group
3173//===----------------------------------------------------------------------===//
3174
3175class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm,
3176                          ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3>
3177: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm),
3178  asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {
3179  bits<5> Zd;
3180  bits<5> Zn;
3181  bits<5> Zm;
3182  let Inst{31-24} = 0b01000101;
3183  let Inst{23-22} = sz;
3184  let Inst{21}    = 0b0;
3185  let Inst{20-16} = Zm;
3186  let Inst{15}    = 0b0;
3187  let Inst{14-10} = opc;
3188  let Inst{9-5}   = Zn;
3189  let Inst{4-0}   = Zd;
3190}
3191
3192multiclass sve2_wide_int_arith_long<bits<5> opc, string asm,
3193                                    SDPatternOperator op> {
3194  def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>;
3195  def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>;
3196  def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>;
3197
3198  def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;
3199  def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;
3200  def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;
3201}
3202
3203multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm,
3204                                    SDPatternOperator op> {
3205  def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>;
3206  def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>;
3207  def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>;
3208
3209  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;
3210  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;
3211  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;
3212}
3213
3214multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm,
3215                                     SDPatternOperator op> {
3216  def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>;
3217
3218  // To avoid using 128 bit elements in the IR, the pattern below works with
3219  // llvm intrinsics with the _pair suffix, to reflect that
3220  // _Q is implemented as a pair of _D.
3221  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;
3222}
3223
3224multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> {
3225  def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>;
3226  def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>;
3227
3228  // To avoid using 128 bit elements in the IR, the patterns below work with
3229  // llvm intrinsics with the _pair suffix, to reflect that
3230  // _H is implemented as a pair of _B and _D is implemented as a pair of _S.
3231  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;
3232  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;
3233}
3234
3235//===----------------------------------------------------------------------===//
3236// SVE2 Misc Group
3237//===----------------------------------------------------------------------===//
3238
3239class sve2_misc<bits<2> sz, bits<4> opc, string asm,
3240                ZPRRegOp zprty1, ZPRRegOp zprty2>
3241: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),
3242  asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {
3243  bits<5> Zd;
3244  bits<5> Zn;
3245  bits<5> Zm;
3246  let Inst{31-24} = 0b01000101;
3247  let Inst{23-22} = sz;
3248  let Inst{21}    = 0b0;
3249  let Inst{20-16} = Zm;
3250  let Inst{15-14} = 0b10;
3251  let Inst{13-10} = opc;
3252  let Inst{9-5}   = Zn;
3253  let Inst{4-0}   = Zd;
3254}
3255
3256multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> {
3257  def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>;
3258  def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>;
3259  def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>;
3260  def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>;
3261
3262  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
3263  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
3264  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
3265  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
3266}
3267
3268multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm,
3269                                                 SDPatternOperator op> {
3270  def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>;
3271  def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>;
3272  def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>;
3273
3274  def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;
3275  def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;
3276  def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;
3277}
3278
3279class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm,
3280                                   ZPRRegOp zprty1, ZPRRegOp zprty2>
3281: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm),
3282  asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {
3283  bits<5> Zd;
3284  bits<5> Zn;
3285  bits<5> Zm;
3286  let Inst{31-24} = 0b01000101;
3287  let Inst{23-22} = sz;
3288  let Inst{21}    = 0b0;
3289  let Inst{20-16} = Zm;
3290  let Inst{15-11} = 0b10010;
3291  let Inst{10}    = opc;
3292  let Inst{9-5}   = Zn;
3293  let Inst{4-0}   = Zd;
3294
3295  let Constraints = "$Zd = $_Zd";
3296  let DestructiveInstType = DestructiveOther;
3297  let ElementSize = ElementSizeNone;
3298}
3299
3300multiclass sve2_bitwise_xor_interleaved<bit opc, string asm,
3301                                        SDPatternOperator op> {
3302  def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8,  ZPR8>;
3303  def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>;
3304  def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>;
3305  def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>;
3306
3307  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
3308  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
3309  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
3310  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
3311}
3312
3313class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm,
3314                                   ZPRRegOp zprty1, ZPRRegOp zprty2,
3315                                   Operand immtype>
3316: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm),
3317  asm, "\t$Zd, $Zn, $imm",
3318  "", []>, Sched<[]> {
3319  bits<5> Zd;
3320  bits<5> Zn;
3321  bits<5> imm;
3322  let Inst{31-23} = 0b010001010;
3323  let Inst{22}    = tsz8_64{2};
3324  let Inst{21}    = 0b0;
3325  let Inst{20-19} = tsz8_64{1-0};
3326  let Inst{18-16} = imm{2-0}; // imm3
3327  let Inst{15-12} = 0b1010;
3328  let Inst{11-10} = opc;
3329  let Inst{9-5}   = Zn;
3330  let Inst{4-0}   = Zd;
3331}
3332
3333multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm,
3334                                        SDPatternOperator op> {
3335  def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm,
3336                                        ZPR16, ZPR8, vecshiftL8>;
3337  def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm,
3338                                        ZPR32, ZPR16, vecshiftL16> {
3339    let Inst{19} = imm{3};
3340  }
3341  def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm,
3342                                        ZPR64, ZPR32, vecshiftL32> {
3343    let Inst{20-19} = imm{4-3};
3344  }
3345  def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8,  !cast<Instruction>(NAME # _H)>;
3346  def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>;
3347  def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>;
3348}
3349
3350//===----------------------------------------------------------------------===//
3351// SVE2 Accumulate Group
3352//===----------------------------------------------------------------------===//
3353
3354class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm,
3355                             ZPRRegOp zprty, Operand immtype>
3356: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm),
3357  asm, "\t$Zd, $Zn, $imm",
3358  "", []>, Sched<[]> {
3359  bits<5> Zd;
3360  bits<5> Zn;
3361  bits<6> imm;
3362  let Inst{31-24} = 0b01000101;
3363  let Inst{23-22} = tsz8_64{3-2};
3364  let Inst{21}    = 0b0;
3365  let Inst{20-19} = tsz8_64{1-0};
3366  let Inst{18-16} = imm{2-0}; // imm3
3367  let Inst{15-11} = 0b11110;
3368  let Inst{10}    = opc;
3369  let Inst{9-5}   = Zn;
3370  let Inst{4-0}   = Zd;
3371
3372  let Constraints = "$Zd = $_Zd";
3373}
3374
3375multiclass sve2_int_bin_shift_imm_left<bit opc, string asm,
3376                                       SDPatternOperator op> {
3377  def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;
3378  def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {
3379    let Inst{19} = imm{3};
3380  }
3381  def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> {
3382    let Inst{20-19} = imm{4-3};
3383  }
3384  def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> {
3385    let Inst{22}    = imm{5};
3386    let Inst{20-19} = imm{4-3};
3387  }
3388
3389  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8,  !cast<Instruction>(NAME # _B)>;
3390  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>;
3391  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>;
3392  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>;
3393}
3394
3395multiclass sve2_int_bin_shift_imm_right<bit opc, string asm,
3396                                        SDPatternOperator op> {
3397  def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;
3398  def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {
3399    let Inst{19} = imm{3};
3400  }
3401  def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {
3402    let Inst{20-19} = imm{4-3};
3403  }
3404  def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {
3405    let Inst{22}    = imm{5};
3406    let Inst{20-19} = imm{4-3};
3407  }
3408
3409  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8,  !cast<Instruction>(NAME # _B)>;
3410  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>;
3411  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>;
3412  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>;
3413}
3414
3415class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm,
3416                                   ZPRRegOp zprty, Operand immtype>
3417: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm),
3418  asm, "\t$Zda, $Zn, $imm",
3419  "", []>, Sched<[]> {
3420  bits<5> Zda;
3421  bits<5> Zn;
3422  bits<6> imm;
3423  let Inst{31-24} = 0b01000101;
3424  let Inst{23-22} = tsz8_64{3-2};
3425  let Inst{21}    = 0b0;
3426  let Inst{20-19} = tsz8_64{1-0};
3427  let Inst{18-16} = imm{2-0}; // imm3
3428  let Inst{15-12} = 0b1110;
3429  let Inst{11-10} = opc;
3430  let Inst{9-5}   = Zn;
3431  let Inst{4-0}   = Zda;
3432
3433  let Constraints = "$Zda = $_Zda";
3434  let DestructiveInstType = DestructiveOther;
3435  let ElementSize = ElementSizeNone;
3436}
3437
3438multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm,
3439                                              SDPatternOperator op> {
3440  def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;
3441  def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {
3442    let Inst{19} = imm{3};
3443  }
3444  def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {
3445    let Inst{20-19} = imm{4-3};
3446  }
3447  def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {
3448    let Inst{22}    = imm{5};
3449    let Inst{20-19} = imm{4-3};
3450  }
3451
3452  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8,  !cast<Instruction>(NAME # _B)>;
3453  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>;
3454  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>;
3455  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>;
3456}
3457
3458class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty>
3459: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot),
3460  asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> {
3461  bits<5> Zdn;
3462  bits<5> Zm;
3463  bit rot;
3464  let Inst{31-24} = 0b01000101;
3465  let Inst{23-22} = sz;
3466  let Inst{21-17} = 0b00000;
3467  let Inst{16}    = opc;
3468  let Inst{15-11} = 0b11011;
3469  let Inst{10}    = rot;
3470  let Inst{9-5}   = Zm;
3471  let Inst{4-0}   = Zdn;
3472
3473  let Constraints = "$Zdn = $_Zdn";
3474  let DestructiveInstType = DestructiveOther;
3475  let ElementSize = ElementSizeNone;
3476}
3477
3478multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> {
3479  def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>;
3480  def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>;
3481  def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>;
3482  def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>;
3483
3484  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>;
3485  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>;
3486  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>;
3487  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>;
3488}
3489
3490class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm,
3491                             ZPRRegOp zprty1, ZPRRegOp zprty2>
3492: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm),
3493  asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
3494  bits<5> Zda;
3495  bits<5> Zn;
3496  bits<5> Zm;
3497  let Inst{31-24} = 0b01000101;
3498  let Inst{23-22} = sz;
3499  let Inst{21}    = 0b0;
3500  let Inst{20-16} = Zm;
3501  let Inst{15-14} = 0b11;
3502  let Inst{13-10} = opc;
3503  let Inst{9-5}   = Zn;
3504  let Inst{4-0}   = Zda;
3505
3506  let Constraints = "$Zda = $_Zda";
3507  let DestructiveInstType = DestructiveOther;
3508  let ElementSize = ElementSizeNone;
3509}
3510
3511multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> {
3512  def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>;
3513  def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>;
3514  def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>;
3515  def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>;
3516
3517  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
3518  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
3519  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
3520  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
3521}
3522
3523multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm,
3524                                       SDPatternOperator op> {
3525  def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>;
3526  def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>;
3527  def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>;
3528
3529  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>;
3530  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>;
3531  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>;
3532}
3533
3534multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm,
3535                                      SDPatternOperator op> {
3536  def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm,
3537                                  ZPR32, ZPR32>;
3538  def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm,
3539                                  ZPR64, ZPR64>;
3540
3541  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
3542  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
3543}
3544
3545//===----------------------------------------------------------------------===//
3546// SVE2 Narrowing Group
3547//===----------------------------------------------------------------------===//
3548
3549class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc,
3550                                           string asm, ZPRRegOp zprty1,
3551                                           ZPRRegOp zprty2, Operand immtype>
3552: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm),
3553  asm, "\t$Zd, $Zn, $imm",
3554  "", []>, Sched<[]> {
3555  bits<5> Zd;
3556  bits<5> Zn;
3557  bits<5> imm;
3558  let Inst{31-23} = 0b010001010;
3559  let Inst{22}    = tsz8_64{2};
3560  let Inst{21}    = 0b1;
3561  let Inst{20-19} = tsz8_64{1-0};
3562  let Inst{18-16} = imm{2-0}; // imm3
3563  let Inst{15-14} = 0b00;
3564  let Inst{13-11} = opc;
3565  let Inst{10}    = 0b0;
3566  let Inst{9-5}   = Zn;
3567  let Inst{4-0}   = Zd;
3568}
3569
3570multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm,
3571                                                      SDPatternOperator op> {
3572  def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16,
3573                                                tvecshiftR8>;
3574  def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32,
3575                                                tvecshiftR16> {
3576    let Inst{19} = imm{3};
3577  }
3578  def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64,
3579                                                tvecshiftR32> {
3580    let Inst{20-19} = imm{4-3};
3581  }
3582  def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8,  !cast<Instruction>(NAME # _B)>;
3583  def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>;
3584  def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>;
3585}
3586
3587class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc,
3588                                        string asm, ZPRRegOp zprty1,
3589                                        ZPRRegOp zprty2, Operand immtype>
3590: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm),
3591  asm, "\t$Zd, $Zn, $imm",
3592  "", []>, Sched<[]> {
3593  bits<5> Zd;
3594  bits<5> Zn;
3595  bits<5> imm;
3596  let Inst{31-23} = 0b010001010;
3597  let Inst{22}    = tsz8_64{2};
3598  let Inst{21}    = 0b1;
3599  let Inst{20-19} = tsz8_64{1-0};
3600  let Inst{18-16} = imm{2-0}; // imm3
3601  let Inst{15-14} = 0b00;
3602  let Inst{13-11} = opc;
3603  let Inst{10}    = 0b1;
3604  let Inst{9-5}   = Zn;
3605  let Inst{4-0}   = Zd;
3606
3607  let Constraints = "$Zd = $_Zd";
3608}
3609
3610multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm,
3611                                                   SDPatternOperator op> {
3612  def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16,
3613                                             tvecshiftR8>;
3614  def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32,
3615                                             tvecshiftR16> {
3616    let Inst{19} = imm{3};
3617  }
3618  def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64,
3619                                             tvecshiftR32> {
3620    let Inst{20-19} = imm{4-3};
3621  }
3622  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8,  !cast<Instruction>(NAME # _B)>;
3623  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>;
3624  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>;
3625}
3626
3627class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm,
3628                                         ZPRRegOp zprty1, ZPRRegOp zprty2>
3629: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm),
3630  asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {
3631  bits<5> Zd;
3632  bits<5> Zn;
3633  bits<5> Zm;
3634  let Inst{31-24} = 0b01000101;
3635  let Inst{23-22} = sz;
3636  let Inst{21}    = 0b1;
3637  let Inst{20-16} = Zm;
3638  let Inst{15-13} = 0b011;
3639  let Inst{12-11} = opc; // S, R
3640  let Inst{10}    = 0b0; // Top
3641  let Inst{9-5}   = Zn;
3642  let Inst{4-0}   = Zd;
3643}
3644
3645multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm,
3646                                              SDPatternOperator op> {
3647  def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>;
3648  def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>;
3649  def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>;
3650
3651  def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>;
3652  def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>;
3653  def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>;
3654}
3655
3656class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm,
3657                                      ZPRRegOp zprty1, ZPRRegOp zprty2>
3658: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm),
3659  asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> {
3660  bits<5> Zd;
3661  bits<5> Zn;
3662  bits<5> Zm;
3663  let Inst{31-24} = 0b01000101;
3664  let Inst{23-22} = sz;
3665  let Inst{21}    = 0b1;
3666  let Inst{20-16} = Zm;
3667  let Inst{15-13} = 0b011;
3668  let Inst{12-11} = opc; // S, R
3669  let Inst{10}    = 0b1; // Top
3670  let Inst{9-5}   = Zn;
3671  let Inst{4-0}   = Zd;
3672
3673  let Constraints = "$Zd = $_Zd";
3674}
3675
3676multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm,
3677                                           SDPatternOperator op> {
3678  def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>;
3679  def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>;
3680  def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>;
3681
3682  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>;
3683  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>;
3684  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>;
3685}
3686
3687class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm,
3688                                         ZPRRegOp zprty1, ZPRRegOp zprty2>
3689: I<(outs zprty1:$Zd), (ins zprty2:$Zn),
3690  asm, "\t$Zd, $Zn", "", []>, Sched<[]> {
3691  bits<5> Zd;
3692  bits<5> Zn;
3693  let Inst{31-23} = 0b010001010;
3694  let Inst{22}    = tsz8_64{2};
3695  let Inst{21}    = 0b1;
3696  let Inst{20-19} = tsz8_64{1-0};
3697  let Inst{18-13} = 0b000010;
3698  let Inst{12-11} = opc;
3699  let Inst{10}    = 0b0;
3700  let Inst{9-5}   = Zn;
3701  let Inst{4-0}   = Zd;
3702}
3703
3704multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm,
3705                                              SDPatternOperator op> {
3706  def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>;
3707  def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>;
3708  def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>;
3709
3710  def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>;
3711  def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>;
3712  def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>;
3713}
3714
3715class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm,
3716                                      ZPRRegOp zprty1, ZPRRegOp zprty2>
3717: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn),
3718  asm, "\t$Zd, $Zn", "", []>, Sched<[]> {
3719  bits<5> Zd;
3720  bits<5> Zn;
3721  let Inst{31-23} = 0b010001010;
3722  let Inst{22}    = tsz8_64{2};
3723  let Inst{21}    = 0b1;
3724  let Inst{20-19} = tsz8_64{1-0};
3725  let Inst{18-13} = 0b000010;
3726  let Inst{12-11} = opc;
3727  let Inst{10}    = 0b1;
3728  let Inst{9-5}   = Zn;
3729  let Inst{4-0}   = Zd;
3730
3731  let Constraints = "$Zd = $_Zd";
3732}
3733
3734multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm,
3735                                           SDPatternOperator op> {
3736  def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>;
3737  def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>;
3738  def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>;
3739
3740  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>;
3741  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>;
3742  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;
3743}
3744
3745//===----------------------------------------------------------------------===//
3746// SVE Integer Arithmetic - Unary Predicated Group
3747//===----------------------------------------------------------------------===//
3748
3749class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc,
3750                             string asm, ZPRRegOp zprty>
3751: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
3752  asm, "\t$Zd, $Pg/m, $Zn",
3753  "",
3754  []>, Sched<[]> {
3755  bits<3> Pg;
3756  bits<5> Zd;
3757  bits<5> Zn;
3758  let Inst{31-24} = 0b00000100;
3759  let Inst{23-22} = sz8_64;
3760  let Inst{21-20} = 0b01;
3761  let Inst{19}    = opc{0};
3762  let Inst{18-16} = opc{3-1};
3763  let Inst{15-13} = 0b101;
3764  let Inst{12-10} = Pg;
3765  let Inst{9-5}   = Zn;
3766  let Inst{4-0}   = Zd;
3767
3768  let Constraints = "$Zd = $_Zd";
3769  let DestructiveInstType = DestructiveOther;
3770  let ElementSize = zprty.ElementSize;
3771}
3772
3773multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm,
3774                                  SDPatternOperator op> {
3775  def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>;
3776  def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>;
3777  def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>;
3778  def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>;
3779
3780  def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
3781  def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
3782  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
3783  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
3784}
3785
3786multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm,
3787                                    SDPatternOperator op> {
3788  def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>;
3789  def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>;
3790  def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>;
3791
3792  def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>;
3793  def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>;
3794  def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>;
3795}
3796
3797multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm,
3798                                    SDPatternOperator op> {
3799  def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>;
3800  def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>;
3801
3802  def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>;
3803  def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>;
3804}
3805
3806multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm,
3807                                    SDPatternOperator op> {
3808  def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>;
3809
3810  def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>;
3811}
3812
3813multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm,
3814                                  SDPatternOperator op> {
3815  def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>;
3816  def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>;
3817  def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>;
3818  def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>;
3819
3820  def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
3821  def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
3822  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
3823  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
3824}
3825
3826multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> {
3827  def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>;
3828  def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>;
3829  def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>;
3830
3831  def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
3832  def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
3833  def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
3834  def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
3835  def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
3836  def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
3837}
3838
3839//===----------------------------------------------------------------------===//
3840// SVE Integer Wide Immediate - Unpredicated Group
3841//===----------------------------------------------------------------------===//
3842class sve_int_dup_imm<bits<2> sz8_64, string asm,
3843                      ZPRRegOp zprty, Operand immtype>
3844: I<(outs zprty:$Zd), (ins immtype:$imm),
3845  asm, "\t$Zd, $imm",
3846  "",
3847  []>, Sched<[]> {
3848  bits<5> Zd;
3849  bits<9> imm;
3850  let Inst{31-24} = 0b00100101;
3851  let Inst{23-22} = sz8_64;
3852  let Inst{21-14} = 0b11100011;
3853  let Inst{13}    = imm{8};   // sh
3854  let Inst{12-5}  = imm{7-0}; // imm8
3855  let Inst{4-0}   = Zd;
3856
3857  let isReMaterializable = 1;
3858}
3859
3860multiclass sve_int_dup_imm<string asm> {
3861  def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>;
3862  def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>;
3863  def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>;
3864  def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>;
3865
3866  def : InstAlias<"mov $Zd, $imm",
3867                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>;
3868  def : InstAlias<"mov $Zd, $imm",
3869                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>;
3870  def : InstAlias<"mov $Zd, $imm",
3871                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>;
3872  def : InstAlias<"mov $Zd, $imm",
3873                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>;
3874
3875  def : InstAlias<"fmov $Zd, #0.0",
3876                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>;
3877  def : InstAlias<"fmov $Zd, #0.0",
3878                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>;
3879  def : InstAlias<"fmov $Zd, #0.0",
3880                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>;
3881}
3882
3883class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype,
3884                        string asm, ZPRRegOp zprty>
3885: I<(outs zprty:$Zd), (ins fpimmtype:$imm8),
3886  asm, "\t$Zd, $imm8",
3887  "",
3888  []>, Sched<[]> {
3889  bits<5> Zd;
3890  bits<8> imm8;
3891  let Inst{31-24} = 0b00100101;
3892  let Inst{23-22} = sz8_64;
3893  let Inst{21-14} = 0b11100111;
3894  let Inst{13}    = 0b0;
3895  let Inst{12-5}  = imm8;
3896  let Inst{4-0}   = Zd;
3897
3898  let isReMaterializable = 1;
3899}
3900
3901multiclass sve_int_dup_fpimm<string asm> {
3902  def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>;
3903  def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>;
3904  def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>;
3905
3906  def : InstAlias<"fmov $Zd, $imm8",
3907                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>;
3908  def : InstAlias<"fmov $Zd, $imm8",
3909                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>;
3910  def : InstAlias<"fmov $Zd, $imm8",
3911                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>;
3912}
3913
3914class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm,
3915                         ZPRRegOp zprty, Operand immtype>
3916: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
3917  asm, "\t$Zdn, $_Zdn, $imm",
3918  "",
3919  []>, Sched<[]> {
3920  bits<5> Zdn;
3921  bits<9> imm;
3922  let Inst{31-24} = 0b00100101;
3923  let Inst{23-22} = sz8_64;
3924  let Inst{21-19} = 0b100;
3925  let Inst{18-16} = opc;
3926  let Inst{15-14} = 0b11;
3927  let Inst{13}    = imm{8};   // sh
3928  let Inst{12-5}  = imm{7-0}; // imm8
3929  let Inst{4-0}   = Zdn;
3930
3931  let Constraints = "$Zdn = $_Zdn";
3932  let DestructiveInstType = DestructiveOther;
3933  let ElementSize = ElementSizeNone;
3934}
3935
3936multiclass sve_int_arith_imm0<bits<3> opc, string asm,
3937                              SDPatternOperator op, SDPatternOperator int_op> {
3938  def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8,  addsub_imm8_opt_lsl_i8>;
3939  def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>;
3940  def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>;
3941  def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>;
3942
3943  def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8,  i32, SVEAddSubImm8Pat,  !cast<Instruction>(NAME # _B)>;
3944  def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>;
3945  def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>;
3946  def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>;
3947
3948  // Intrinsic version
3949  def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, int_op, ZPR8,  i32, SVEAddSubImm8Pat,  !cast<Instruction>(NAME # _B)>;
3950  def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, int_op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>;
3951  def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, int_op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>;
3952  def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, int_op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>;
3953}
3954
3955multiclass sve_int_arith_imm0_subr<bits<3> opc, string asm, SDPatternOperator op> {
3956  def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8,  addsub_imm8_opt_lsl_i8>;
3957  def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>;
3958  def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>;
3959  def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>;
3960
3961  def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv16i8, op, ZPR8,  i32, SVEAddSubImm8Pat,  !cast<Instruction>(NAME # _B)>;
3962  def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>;
3963  def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>;
3964  def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>;
3965}
3966
3967class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm,
3968                        ZPRRegOp zprty, Operand immtype>
3969: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
3970  asm, "\t$Zdn, $_Zdn, $imm",
3971  "",
3972  []>, Sched<[]> {
3973  bits<5> Zdn;
3974  bits<8> imm;
3975  let Inst{31-24} = 0b00100101;
3976  let Inst{23-22} = sz8_64;
3977  let Inst{21-16} = opc;
3978  let Inst{15-13} = 0b110;
3979  let Inst{12-5} = imm;
3980  let Inst{4-0} = Zdn;
3981
3982  let Constraints = "$Zdn = $_Zdn";
3983  let DestructiveInstType = DestructiveOther;
3984  let ElementSize = ElementSizeNone;
3985}
3986
3987multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> {
3988  def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>;
3989  def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>;
3990  def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>;
3991  def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>;
3992
3993  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>;
3994  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1,  op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>;
3995  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1,  op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>;
3996  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1,  op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>;
3997}
3998
3999multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> {
4000  def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>;
4001  def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>;
4002  def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>;
4003  def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>;
4004
4005  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>;
4006  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>;
4007  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>;
4008  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>;
4009}
4010
4011multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> {
4012  def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8,  simm8>;
4013  def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>;
4014  def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>;
4015  def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>;
4016
4017  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>;
4018  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>;
4019  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>;
4020  def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>;
4021}
4022
4023//===----------------------------------------------------------------------===//
4024// SVE Bitwise Logical - Unpredicated Group
4025//===----------------------------------------------------------------------===//
4026
4027class sve_int_bin_cons_log<bits<2> opc, string asm>
4028: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm),
4029  asm, "\t$Zd, $Zn, $Zm",
4030  "",
4031  []>, Sched<[]> {
4032  bits<5> Zd;
4033  bits<5> Zm;
4034  bits<5> Zn;
4035  let Inst{31-24} = 0b00000100;
4036  let Inst{23-22} = opc{1-0};
4037  let Inst{21}    = 0b1;
4038  let Inst{20-16} = Zm;
4039  let Inst{15-10} = 0b001100;
4040  let Inst{9-5}   = Zn;
4041  let Inst{4-0}   = Zd;
4042}
4043
4044multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> {
4045  def NAME : sve_int_bin_cons_log<opc, asm>;
4046
4047  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
4048  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;
4049  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;
4050  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;
4051
4052  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
4053                  (!cast<Instruction>(NAME) ZPR8:$Zd,  ZPR8:$Zn,  ZPR8:$Zm),  1>;
4054  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
4055                  (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>;
4056  def : InstAlias<asm # "\t$Zd, $Zn, $Zm",
4057                  (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>;
4058}
4059
4060class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm>
4061: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk),
4062  asm, "\t$Zdn, $_Zdn, $Zm, $Zk",
4063  "",
4064  []>, Sched<[]> {
4065  bits<5> Zdn;
4066  bits<5> Zk;
4067  bits<5> Zm;
4068  let Inst{31-24} = 0b00000100;
4069  let Inst{23-22} = opc{2-1};
4070  let Inst{21}    = 0b1;
4071  let Inst{20-16} = Zm;
4072  let Inst{15-11} = 0b00111;
4073  let Inst{10}    = opc{0};
4074  let Inst{9-5}   = Zk;
4075  let Inst{4-0}   = Zdn;
4076
4077  let Constraints = "$Zdn = $_Zdn";
4078  let DestructiveInstType = DestructiveOther;
4079  let ElementSize = ElementSizeNone;
4080}
4081
4082multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op> {
4083  def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>;
4084
4085  def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
4086                  (!cast<Instruction>(NAME) ZPR8:$Zdn,  ZPR8:$Zm,  ZPR8:$Zk),  1>;
4087  def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
4088                  (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>;
4089  def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk",
4090                  (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>;
4091
4092  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
4093  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;
4094  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>;
4095  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>;
4096}
4097
4098class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm,
4099                                ZPRRegOp zprty, Operand immtype>
4100: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm),
4101  asm, "\t$Zdn, $_Zdn, $Zm, $imm",
4102  "",
4103  []>, Sched<[]> {
4104  bits<5> Zdn;
4105  bits<5> Zm;
4106  bits<6> imm;
4107  let Inst{31-24} = 0b00000100;
4108  let Inst{23-22} = tsz8_64{3-2};
4109  let Inst{21}    = 0b1;
4110  let Inst{20-19} = tsz8_64{1-0};
4111  let Inst{18-16} = imm{2-0}; // imm3
4112  let Inst{15-10} = 0b001101;
4113  let Inst{9-5}   = Zm;
4114  let Inst{4-0}   = Zdn;
4115
4116  let Constraints = "$Zdn = $_Zdn";
4117  let DestructiveInstType = DestructiveOther;
4118  let ElementSize = ElementSizeNone;
4119}
4120
4121multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> {
4122  def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>;
4123  def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> {
4124    let Inst{19} = imm{3};
4125  }
4126  def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> {
4127    let Inst{20-19} = imm{4-3};
4128  }
4129  def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> {
4130    let Inst{22}    = imm{5};
4131    let Inst{20-19} = imm{4-3};
4132  }
4133
4134  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8,  !cast<Instruction>(NAME # _B)>;
4135  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>;
4136  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>;
4137  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>;
4138}
4139
4140//===----------------------------------------------------------------------===//
4141// SVE Integer Wide Immediate - Predicated Group
4142//===----------------------------------------------------------------------===//
4143
4144class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype,
4145                             string asm, ZPRRegOp zprty>
4146: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8),
4147  asm, "\t$Zd, $Pg/m, $imm8",
4148  "",
4149  []>, Sched<[]> {
4150  bits<4> Pg;
4151  bits<5> Zd;
4152  bits<8> imm8;
4153  let Inst{31-24} = 0b00000101;
4154  let Inst{23-22} = sz;
4155  let Inst{21-20} = 0b01;
4156  let Inst{19-16} = Pg;
4157  let Inst{15-13} = 0b110;
4158  let Inst{12-5}  = imm8;
4159  let Inst{4-0}   = Zd;
4160
4161  let Constraints = "$Zd = $_Zd";
4162  let DestructiveInstType = DestructiveOther;
4163  let ElementSize = zprty.ElementSize;
4164}
4165
4166multiclass sve_int_dup_fpimm_pred<string asm> {
4167  def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>;
4168  def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>;
4169  def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>;
4170
4171  def : InstAlias<"fmov $Zd, $Pg/m, $imm8",
4172                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>;
4173  def : InstAlias<"fmov $Zd, $Pg/m, $imm8",
4174                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>;
4175  def : InstAlias<"fmov $Zd, $Pg/m, $imm8",
4176                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>;
4177}
4178
4179class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm,
4180                           ZPRRegOp zprty, string pred_qual, dag iops>
4181: I<(outs zprty:$Zd), iops,
4182  asm, "\t$Zd, $Pg"#pred_qual#", $imm",
4183  "", []>, Sched<[]> {
4184  bits<5> Zd;
4185  bits<4> Pg;
4186  bits<9> imm;
4187  let Inst{31-24} = 0b00000101;
4188  let Inst{23-22} = sz8_64;
4189  let Inst{21-20} = 0b01;
4190  let Inst{19-16} = Pg;
4191  let Inst{15}    = 0b0;
4192  let Inst{14}    = m;
4193  let Inst{13}    = imm{8};   // sh
4194  let Inst{12-5}  = imm{7-0}; // imm8
4195  let Inst{4-0}   = Zd;
4196
4197  let DestructiveInstType = DestructiveOther;
4198  let ElementSize = zprty.ElementSize;
4199}
4200
4201multiclass sve_int_dup_imm_pred_merge_inst<
4202    bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty,
4203    ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> {
4204  let Constraints = "$Zd = $_Zd" in
4205  def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty,  "/m",
4206                                  (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>;
4207  def : InstAlias<"mov $Zd, $Pg/m, $imm",
4208                  (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>;
4209  def : Pat<(intty
4210              (vselect predty:$Pg,
4211                (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))),
4212                intty:$Zd)),
4213            (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>;
4214}
4215
4216multiclass sve_int_dup_imm_pred_merge<string asm> {
4217  defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1,
4218                                            i32, cpy_imm8_opt_lsl_i8>;
4219  defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1,
4220                                            i32, cpy_imm8_opt_lsl_i16>;
4221  defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1,
4222                                            i32, cpy_imm8_opt_lsl_i32>;
4223  defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1,
4224                                            i64, cpy_imm8_opt_lsl_i64>;
4225
4226  def : InstAlias<"fmov $Zd, $Pg/m, #0.0",
4227                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>;
4228  def : InstAlias<"fmov $Zd, $Pg/m, #0.0",
4229                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>;
4230  def : InstAlias<"fmov $Zd, $Pg/m, #0.0",
4231                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>;
4232}
4233
4234multiclass sve_int_dup_imm_pred_zero_inst<
4235    bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty,
4236    ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> {
4237  def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z",
4238                                  (ins PPRAny:$Pg, cpyimm:$imm)>;
4239  def : InstAlias<"mov $Zd, $Pg/z, $imm",
4240                  (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>;
4241  def : Pat<(intty (zext (predty PPRAny:$Ps1))),
4242            (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>;
4243  def : Pat<(intty (sext (predty PPRAny:$Ps1))),
4244            (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>;
4245  def : Pat<(intty (anyext (predty PPRAny:$Ps1))),
4246            (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>;
4247  def : Pat<(intty
4248              (vselect predty:$Pg,
4249                (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))),
4250                (intty (AArch64dup (scalarty 0))))),
4251            (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>;
4252}
4253
4254multiclass sve_int_dup_imm_pred_zero<string asm> {
4255  defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8,  nxv16i8, nxv16i1,
4256                                           i32, cpy_imm8_opt_lsl_i8>;
4257  defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1,
4258                                           i32, cpy_imm8_opt_lsl_i16>;
4259  defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1,
4260                                           i32, cpy_imm8_opt_lsl_i32>;
4261  defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1,
4262                                           i64, cpy_imm8_opt_lsl_i64>;
4263}
4264
4265//===----------------------------------------------------------------------===//
4266// SVE Integer Compare - Vectors Group
4267//===----------------------------------------------------------------------===//
4268
4269class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm,
4270                  PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2>
4271: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm),
4272  asm, "\t$Pd, $Pg/z, $Zn, $Zm",
4273  "",
4274  []>, Sched<[]> {
4275  bits<4> Pd;
4276  bits<3> Pg;
4277  bits<5> Zm;
4278  bits<5> Zn;
4279  let Inst{31-24} = 0b00100100;
4280  let Inst{23-22} = sz8_64;
4281  let Inst{21}    = 0b0;
4282  let Inst{20-16} = Zm;
4283  let Inst{15}    = opc{2};
4284  let Inst{14}    = cmp_1;
4285  let Inst{13}    = opc{1};
4286  let Inst{12-10} = Pg;
4287  let Inst{9-5}   = Zn;
4288  let Inst{4}     = opc{0};
4289  let Inst{3-0}   = Pd;
4290
4291  let Defs = [NZCV];
4292  let ElementSize = pprty.ElementSize;
4293  let isPTestLike = 1;
4294}
4295
4296multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt,
4297                         ValueType intvt, sve_int_cmp cmp> {
4298  def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)),
4299            (cmp $Op1, $Op2, $Op3)>;
4300  def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)),
4301            (cmp $Op1, $Op3, $Op2)>;
4302}
4303
4304multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> {
4305  def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>;
4306  def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>;
4307  def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>;
4308  def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>;
4309
4310  defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
4311  defm : SVE_SETCC_Pat<cc, invcc, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
4312  defm : SVE_SETCC_Pat<cc, invcc, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
4313  defm : SVE_SETCC_Pat<cc, invcc, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
4314}
4315
4316multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> {
4317  def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>;
4318  def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>;
4319  def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>;
4320
4321  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;
4322  def : SVE_3_Op_Pat<nxv8i1,  op, nxv8i1,  nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;
4323  def : SVE_3_Op_Pat<nxv4i1,  op, nxv4i1,  nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;
4324}
4325
4326multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> {
4327  def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>;
4328  def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>;
4329  def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>;
4330
4331  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;
4332  def : SVE_3_Op_Pat<nxv8i1,  op, nxv8i1,  nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;
4333  def : SVE_3_Op_Pat<nxv4i1,  op, nxv4i1,  nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;
4334}
4335
4336
4337//===----------------------------------------------------------------------===//
4338// SVE Integer Compare - Signed Immediate Group
4339//===----------------------------------------------------------------------===//
4340
4341class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty,
4342                      ZPRRegOp zprty,
4343                      Operand immtype>
4344: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5),
4345  asm, "\t$Pd, $Pg/z, $Zn, $imm5",
4346  "",
4347  []>, Sched<[]> {
4348  bits<4> Pd;
4349  bits<3> Pg;
4350  bits<5> Zn;
4351  bits<5> imm5;
4352  let Inst{31-24} = 0b00100101;
4353  let Inst{23-22} = sz8_64;
4354  let Inst{21}    = 0b0;
4355  let Inst{20-16} = imm5;
4356  let Inst{15}    = opc{2};
4357  let Inst{14}    = 0b0;
4358  let Inst{13}    = opc{1};
4359  let Inst{12-10} = Pg;
4360  let Inst{9-5}   = Zn;
4361  let Inst{4}     = opc{0};
4362  let Inst{3-0}   = Pd;
4363
4364  let Defs = [NZCV];
4365  let ElementSize = pprty.ElementSize;
4366  let isPTestLike = 1;
4367}
4368
4369multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc,
4370                             ValueType predvt, ValueType intvt,
4371                             Operand immtype, Instruction cmp> {
4372  def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),
4373                                       (intvt ZPR:$Zs1),
4374                                       (intvt (AArch64dup (immtype:$imm))),
4375                                       cc)),
4376            (cmp $Pg, $Zs1, immtype:$imm)>;
4377  def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg),
4378                                       (intvt (AArch64dup (immtype:$imm))),
4379                                       (intvt ZPR:$Zs1),
4380                                       commuted_cc)),
4381            (cmp $Pg, $Zs1, immtype:$imm)>;
4382}
4383
4384multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> {
4385  def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>;
4386  def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>;
4387  def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>;
4388  def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>;
4389
4390  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b,
4391                           !cast<Instruction>(NAME # _B)>;
4392  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1,  nxv8i16, simm5_32b,
4393                           !cast<Instruction>(NAME # _H)>;
4394  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1,  nxv4i32, simm5_32b,
4395                           !cast<Instruction>(NAME # _S)>;
4396  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1,  nxv2i64, simm5_64b,
4397                           !cast<Instruction>(NAME # _D)>;
4398}
4399
4400
4401//===----------------------------------------------------------------------===//
4402// SVE Integer Compare - Unsigned Immediate Group
4403//===----------------------------------------------------------------------===//
4404
4405class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty,
4406                      ZPRRegOp zprty, Operand immtype>
4407: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7),
4408  asm, "\t$Pd, $Pg/z, $Zn, $imm7",
4409  "",
4410  []>, Sched<[]> {
4411  bits<4> Pd;
4412  bits<3> Pg;
4413  bits<5> Zn;
4414  bits<7> imm7;
4415  let Inst{31-24} = 0b00100100;
4416  let Inst{23-22} = sz8_64;
4417  let Inst{21}    = 1;
4418  let Inst{20-14} = imm7;
4419  let Inst{13}    = opc{1};
4420  let Inst{12-10} = Pg;
4421  let Inst{9-5}   = Zn;
4422  let Inst{4}     = opc{0};
4423  let Inst{3-0}   = Pd;
4424
4425  let Defs = [NZCV];
4426  let ElementSize = pprty.ElementSize;
4427  let isPTestLike = 1;
4428}
4429
4430multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc,
4431                           CondCode commuted_cc> {
4432  def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>;
4433  def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>;
4434  def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>;
4435  def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>;
4436
4437  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127,
4438                           !cast<Instruction>(NAME # _B)>;
4439  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1,  nxv8i16, imm0_127,
4440                           !cast<Instruction>(NAME # _H)>;
4441  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1,  nxv4i32, imm0_127,
4442                           !cast<Instruction>(NAME # _S)>;
4443  defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1,  nxv2i64, imm0_127_64b,
4444                           !cast<Instruction>(NAME # _D)>;
4445}
4446
4447
4448//===----------------------------------------------------------------------===//
4449// SVE Integer Compare - Scalars Group
4450//===----------------------------------------------------------------------===//
4451
4452class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt>
4453: I<(outs), (ins rt:$Rn, rt:$Rm),
4454  asm, "\t$Rn, $Rm",
4455  "",
4456  []>, Sched<[]> {
4457  bits<5> Rm;
4458  bits<5> Rn;
4459  let Inst{31-23} = 0b001001011;
4460  let Inst{22}    = sz;
4461  let Inst{21}    = 0b1;
4462  let Inst{20-16} = Rm;
4463  let Inst{15-10} = 0b001000;
4464  let Inst{9-5}   = Rn;
4465  let Inst{4}     = opc;
4466  let Inst{3-0}   = 0b0000;
4467
4468  let Defs = [NZCV];
4469}
4470
4471class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm,
4472                       RegisterClass gprty, PPRRegOp pprty>
4473: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm),
4474  asm, "\t$Pd, $Rn, $Rm",
4475  "", []>, Sched<[]> {
4476  bits<4> Pd;
4477  bits<5> Rm;
4478  bits<5> Rn;
4479  let Inst{31-24} = 0b00100101;
4480  let Inst{23-22} = sz8_64;
4481  let Inst{21}    = 0b1;
4482  let Inst{20-16} = Rm;
4483  let Inst{15-13} = 0b000;
4484  let Inst{12-10} = opc{3-1};
4485  let Inst{9-5}   = Rn;
4486  let Inst{4}     = opc{0};
4487  let Inst{3-0}   = Pd;
4488
4489  let Defs = [NZCV];
4490  let ElementSize = pprty.ElementSize;
4491  let isWhile = 1;
4492}
4493
4494multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> {
4495  def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>;
4496  def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>;
4497  def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>;
4498  def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>;
4499
4500  def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>;
4501  def : SVE_2_Op_Pat<nxv8i1,  op, i32, i32, !cast<Instruction>(NAME # _H)>;
4502  def : SVE_2_Op_Pat<nxv4i1,  op, i32, i32, !cast<Instruction>(NAME # _S)>;
4503  def : SVE_2_Op_Pat<nxv2i1,  op, i32, i32, !cast<Instruction>(NAME # _D)>;
4504}
4505
4506multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> {
4507  def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>;
4508  def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>;
4509  def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>;
4510  def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>;
4511
4512  def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>;
4513  def : SVE_2_Op_Pat<nxv8i1,  op, i64, i64, !cast<Instruction>(NAME # _H)>;
4514  def : SVE_2_Op_Pat<nxv4i1,  op, i64, i64, !cast<Instruction>(NAME # _S)>;
4515  def : SVE_2_Op_Pat<nxv2i1,  op, i64, i64, !cast<Instruction>(NAME # _D)>;
4516}
4517
4518class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm,
4519                        PPRRegOp pprty>
4520: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm),
4521  asm, "\t$Pd, $Rn, $Rm",
4522  "", []>, Sched<[]> {
4523  bits<4> Pd;
4524  bits<5> Rm;
4525  bits<5> Rn;
4526  let Inst{31-24} = 0b00100101;
4527  let Inst{23-22} = sz8_64;
4528  let Inst{21}    = 0b1;
4529  let Inst{20-16} = Rm;
4530  let Inst{15-10} = 0b001100;
4531  let Inst{9-5}   = Rn;
4532  let Inst{4}     = rw;
4533  let Inst{3-0}   = Pd;
4534
4535  let Defs = [NZCV];
4536  let ElementSize = pprty.ElementSize;
4537  let isWhile = 1;
4538}
4539
4540multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> {
4541  def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>;
4542  def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>;
4543  def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>;
4544  def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>;
4545
4546  def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>;
4547  def : SVE_2_Op_Pat<nxv8i1,  !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>;
4548  def : SVE_2_Op_Pat<nxv4i1,  !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>;
4549  def : SVE_2_Op_Pat<nxv2i1,  !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>;
4550}
4551
4552//===----------------------------------------------------------------------===//
4553// SVE Floating Point Fast Reduction Group
4554//===----------------------------------------------------------------------===//
4555
4556class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm,
4557                      ZPRRegOp zprty, FPRasZPROperand dstOpType>
4558: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),
4559  asm, "\t$Vd, $Pg, $Zn",
4560  "",
4561  []>, Sched<[]> {
4562  bits<5> Zn;
4563  bits<5> Vd;
4564  bits<3> Pg;
4565  let Inst{31-24} = 0b01100101;
4566  let Inst{23-22} = sz;
4567  let Inst{21-19} = 0b000;
4568  let Inst{18-16} = opc;
4569  let Inst{15-13} = 0b001;
4570  let Inst{12-10} = Pg;
4571  let Inst{9-5}   = Zn;
4572  let Inst{4-0}   = Vd;
4573}
4574
4575multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> {
4576  def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>;
4577  def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>;
4578  def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>;
4579
4580  def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
4581  def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
4582  def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
4583  def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
4584  def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
4585  def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
4586}
4587
4588//===----------------------------------------------------------------------===//
4589// SVE Floating Point Accumulating Reduction Group
4590//===----------------------------------------------------------------------===//
4591
4592class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm,
4593                      ZPRRegOp zprty, FPRasZPROperand dstOpType>
4594: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm),
4595  asm, "\t$Vdn, $Pg, $_Vdn, $Zm",
4596  "",
4597  []>,
4598  Sched<[]> {
4599  bits<3> Pg;
4600  bits<5> Vdn;
4601  bits<5> Zm;
4602  let Inst{31-24} = 0b01100101;
4603  let Inst{23-22} = sz;
4604  let Inst{21-19} = 0b011;
4605  let Inst{18-16} = opc;
4606  let Inst{15-13} = 0b001;
4607  let Inst{12-10} = Pg;
4608  let Inst{9-5}   = Zm;
4609  let Inst{4-0}   = Vdn;
4610
4611  let Constraints = "$Vdn = $_Vdn";
4612}
4613
4614multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> {
4615  def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>;
4616  def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>;
4617  def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>;
4618
4619  def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>;
4620  def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>;
4621  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
4622  def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>;
4623  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
4624  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
4625}
4626
4627//===----------------------------------------------------------------------===//
4628// SVE Floating Point Compare - Vectors Group
4629//===----------------------------------------------------------------------===//
4630
4631class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty,
4632                      ZPRRegOp zprty>
4633: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),
4634  asm, "\t$Pd, $Pg/z, $Zn, $Zm",
4635  "",
4636  []>, Sched<[]> {
4637  bits<4> Pd;
4638  bits<3> Pg;
4639  bits<5> Zm;
4640  bits<5> Zn;
4641  let Inst{31-24} = 0b01100101;
4642  let Inst{23-22} = sz;
4643  let Inst{21}    = 0b0;
4644  let Inst{20-16} = Zm;
4645  let Inst{15}    = opc{2};
4646  let Inst{14}    = 0b1;
4647  let Inst{13}    = opc{1};
4648  let Inst{12-10} = Pg;
4649  let Inst{9-5}   = Zn;
4650  let Inst{4}     = opc{0};
4651  let Inst{3-0}   = Pd;
4652}
4653
4654multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> {
4655  def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>;
4656  def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>;
4657  def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>;
4658
4659  def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
4660  def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
4661  def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
4662}
4663
4664multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, SDPatternOperator op,
4665                              SDPatternOperator op_nopred>
4666: sve_fp_3op_p_pd<opc, asm, op> {
4667  def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8f16, nxv8f16,
4668                               !cast<Instruction>(NAME # _H), PTRUE_H>;
4669  def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f16, nxv4f16,
4670                               !cast<Instruction>(NAME # _H), PTRUE_S>;
4671  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f16, nxv2f16,
4672                               !cast<Instruction>(NAME # _H), PTRUE_D>;
4673  def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f32, nxv4f32,
4674                               !cast<Instruction>(NAME # _S), PTRUE_S>;
4675  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f32, nxv2f32,
4676                               !cast<Instruction>(NAME # _S), PTRUE_D>;
4677  def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f64, nxv2f64,
4678                               !cast<Instruction>(NAME # _D), PTRUE_D>;
4679}
4680
4681//===----------------------------------------------------------------------===//
4682// SVE Floating Point Compare - with Zero Group
4683//===----------------------------------------------------------------------===//
4684
4685class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty,
4686                      ZPRRegOp zprty>
4687: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn),
4688  asm, "\t$Pd, $Pg/z, $Zn, #0.0",
4689  "",
4690  []>, Sched<[]> {
4691  bits<4> Pd;
4692  bits<3> Pg;
4693  bits<5> Zn;
4694  let Inst{31-24} = 0b01100101;
4695  let Inst{23-22} = sz;
4696  let Inst{21-18} = 0b0100;
4697  let Inst{17-16} = opc{2-1};
4698  let Inst{15-13} = 0b001;
4699  let Inst{12-10} = Pg;
4700  let Inst{9-5}   = Zn;
4701  let Inst{4}     = opc{0};
4702  let Inst{3-0}   = Pd;
4703}
4704
4705multiclass sve_fp_2op_p_pd<bits<3> opc, string asm> {
4706  def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>;
4707  def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>;
4708  def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>;
4709}
4710
4711
4712//===----------------------------------------------------------------------===//
4713//SVE Index Generation Group
4714//===----------------------------------------------------------------------===//
4715
4716class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty,
4717                       Operand imm_ty>
4718: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b),
4719  asm, "\t$Zd, $imm5, $imm5b",
4720  "", []>, Sched<[]> {
4721  bits<5> Zd;
4722  bits<5> imm5;
4723  bits<5> imm5b;
4724  let Inst{31-24} = 0b00000100;
4725  let Inst{23-22} = sz8_64;
4726  let Inst{21}    = 0b1;
4727  let Inst{20-16} = imm5b;
4728  let Inst{15-10} = 0b010000;
4729  let Inst{9-5}   = imm5;
4730  let Inst{4-0}   = Zd;
4731}
4732
4733multiclass sve_int_index_ii<string asm, SDPatternOperator op> {
4734  def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>;
4735  def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>;
4736  def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>;
4737  def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>;
4738
4739  def : Pat<(nxv16i8 (op simm5_8b:$imm5, simm5_8b:$imm5b)),
4740            (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, simm5_8b:$imm5b)>;
4741  def : Pat<(nxv8i16 (op simm5_16b:$imm5, simm5_16b:$imm5b)),
4742            (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, simm5_16b:$imm5b)>;
4743  def : Pat<(nxv4i32 (op simm5_32b:$imm5, simm5_32b:$imm5b)),
4744            (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>;
4745  def : Pat<(nxv2i64 (op simm5_64b:$imm5, simm5_64b:$imm5b)),
4746            (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>;
4747}
4748
4749class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty,
4750                       RegisterClass srcRegType, Operand imm_ty>
4751: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm),
4752  asm, "\t$Zd, $imm5, $Rm",
4753  "", []>, Sched<[]> {
4754  bits<5> Rm;
4755  bits<5> Zd;
4756  bits<5> imm5;
4757  let Inst{31-24} = 0b00000100;
4758  let Inst{23-22} = sz8_64;
4759  let Inst{21}    = 0b1;
4760  let Inst{20-16} = Rm;
4761  let Inst{15-10} = 0b010010;
4762  let Inst{9-5}   = imm5;
4763  let Inst{4-0}   = Zd;
4764}
4765
4766multiclass sve_int_index_ir<string asm, SDPatternOperator op> {
4767  def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>;
4768  def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>;
4769  def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>;
4770  def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>;
4771
4772  def : Pat<(nxv16i8 (op simm5_8b:$imm5, GPR32:$Rm)),
4773            (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>;
4774  def : Pat<(nxv8i16 (op simm5_16b:$imm5, GPR32:$Rm)),
4775            (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>;
4776  def : Pat<(nxv4i32 (op simm5_32b:$imm5, GPR32:$Rm)),
4777            (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>;
4778  def : Pat<(nxv2i64 (op simm5_64b:$imm5, GPR64:$Rm)),
4779            (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>;
4780}
4781
4782class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty,
4783                       RegisterClass srcRegType, Operand imm_ty>
4784: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5),
4785  asm, "\t$Zd, $Rn, $imm5",
4786  "", []>, Sched<[]> {
4787  bits<5> Rn;
4788  bits<5> Zd;
4789  bits<5> imm5;
4790  let Inst{31-24} = 0b00000100;
4791  let Inst{23-22} = sz8_64;
4792  let Inst{21}    = 0b1;
4793  let Inst{20-16} = imm5;
4794  let Inst{15-10} = 0b010001;
4795  let Inst{9-5}   = Rn;
4796  let Inst{4-0}   = Zd;
4797}
4798
4799multiclass sve_int_index_ri<string asm, SDPatternOperator op> {
4800  def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>;
4801  def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>;
4802  def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>;
4803  def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>;
4804
4805  def : Pat<(nxv16i8 (op GPR32:$Rm, simm5_8b:$imm5)),
4806            (!cast<Instruction>(NAME # "_B") GPR32:$Rm, simm5_8b:$imm5)>;
4807  def : Pat<(nxv8i16 (op GPR32:$Rm, simm5_16b:$imm5)),
4808            (!cast<Instruction>(NAME # "_H") GPR32:$Rm, simm5_16b:$imm5)>;
4809  def : Pat<(nxv4i32 (op GPR32:$Rm, simm5_32b:$imm5)),
4810            (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>;
4811  def : Pat<(nxv2i64 (op GPR64:$Rm, simm5_64b:$imm5)),
4812            (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>;
4813}
4814
4815class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty,
4816                       RegisterClass srcRegType>
4817: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm),
4818  asm, "\t$Zd, $Rn, $Rm",
4819  "", []>, Sched<[]> {
4820  bits<5> Zd;
4821  bits<5> Rm;
4822  bits<5> Rn;
4823  let Inst{31-24} = 0b00000100;
4824  let Inst{23-22} = sz8_64;
4825  let Inst{21}    = 0b1;
4826  let Inst{20-16} = Rm;
4827  let Inst{15-10} = 0b010011;
4828  let Inst{9-5}   = Rn;
4829  let Inst{4-0}   = Zd;
4830}
4831
4832multiclass sve_int_index_rr<string asm, SDPatternOperator op> {
4833  def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>;
4834  def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>;
4835  def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>;
4836  def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>;
4837
4838  def : SVE_2_Op_Pat<nxv16i8, op, i32, i32, !cast<Instruction>(NAME # _B)>;
4839  def : SVE_2_Op_Pat<nxv8i16, op, i32, i32, !cast<Instruction>(NAME # _H)>;
4840  def : SVE_2_Op_Pat<nxv4i32, op, i32, i32, !cast<Instruction>(NAME # _S)>;
4841  def : SVE_2_Op_Pat<nxv2i64, op, i64, i64, !cast<Instruction>(NAME # _D)>;
4842}
4843
4844//===----------------------------------------------------------------------===//
4845// SVE Bitwise Shift - Predicated Group
4846//===----------------------------------------------------------------------===//
4847
4848class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm,
4849                                 ZPRRegOp zprty, Operand immtype>
4850: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm),
4851  asm, "\t$Zdn, $Pg/m, $_Zdn, $imm",
4852  "",
4853  []>, Sched<[]> {
4854  bits<3> Pg;
4855  bits<5> Zdn;
4856  bits<6> imm;
4857  let Inst{31-24} = 0b00000100;
4858  let Inst{23-22} = tsz8_64{3-2};
4859  let Inst{21-20} = 0b00;
4860  let Inst{19-16} = opc;
4861  let Inst{15-13} = 0b100;
4862  let Inst{12-10} = Pg;
4863  let Inst{9-8}   = tsz8_64{1-0};
4864  let Inst{7-5}   = imm{2-0}; // imm3
4865  let Inst{4-0}   = Zdn;
4866
4867  let Constraints = "$Zdn = $_Zdn";
4868  let DestructiveInstType = DestructiveBinaryImm;
4869  let ElementSize = zprty.ElementSize;
4870}
4871
4872multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps,
4873                                           SDPatternOperator op = null_frag> {
4874  def _B : SVEPseudo2Instr<Ps # _B, 1>,
4875           sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;
4876  def _H : SVEPseudo2Instr<Ps # _H, 1>,
4877           sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {
4878    let Inst{8} = imm{3};
4879  }
4880  def _S : SVEPseudo2Instr<Ps # _S, 1>,
4881           sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> {
4882    let Inst{9-8} = imm{4-3};
4883  }
4884  def _D : SVEPseudo2Instr<Ps # _D, 1>,
4885           sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> {
4886    let Inst{22}  = imm{5};
4887    let Inst{9-8} = imm{4-3};
4888  }
4889
4890  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8,  !cast<Instruction>(NAME # _B)>;
4891  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1,  nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>;
4892  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1,  nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>;
4893  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1,  nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>;
4894}
4895
4896// As above but shift amount takes the form of a "vector immediate".
4897multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm,
4898                                               string Ps, SDPatternOperator op>
4899: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> {
4900  def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8,  !cast<Instruction>(NAME # _B)>;
4901  def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1,  i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>;
4902  def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1,  i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>;
4903  def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1,  i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>;
4904}
4905
4906multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> {
4907  def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8,  tvecshiftL8,  FalseLanesZero>;
4908  def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>;
4909  def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>;
4910  def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>;
4911
4912  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8,  !cast<Pseudo>(NAME # _ZERO_B)>;
4913  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1,  nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>;
4914  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1,  nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>;
4915  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1,  nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>;
4916}
4917
4918multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps,
4919                                            SDPatternOperator op = null_frag> {
4920  def _B : SVEPseudo2Instr<Ps # _B, 1>,
4921           sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;
4922  def _H : SVEPseudo2Instr<Ps # _H, 1>,
4923           sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {
4924    let Inst{8} = imm{3};
4925  }
4926  def _S : SVEPseudo2Instr<Ps # _S, 1>,
4927           sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {
4928    let Inst{9-8} = imm{4-3};
4929  }
4930  def _D : SVEPseudo2Instr<Ps # _D, 1>,
4931           sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {
4932    let Inst{22}  = imm{5};
4933    let Inst{9-8} = imm{4-3};
4934  }
4935
4936  def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8,  !cast<Instruction>(NAME # _B)>;
4937  def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1,  nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>;
4938  def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1,  nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>;
4939  def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1,  nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>;
4940}
4941
4942// As above but shift amount takes the form of a "vector immediate".
4943multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm,
4944                                            string Ps, SDPatternOperator op>
4945: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> {
4946  def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8,  !cast<Instruction>(NAME # _B)>;
4947  def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1,  i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>;
4948  def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1,  i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>;
4949  def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1,  i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>;
4950}
4951
4952multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> {
4953  def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>;
4954  def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>;
4955  def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>;
4956  def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>;
4957
4958  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>;
4959  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>;
4960  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>;
4961  def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>;
4962}
4963
4964class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc,
4965                             string asm, ZPRRegOp zprty, ZPRRegOp zprty2>
4966: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm),
4967  asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm",
4968  "",
4969  []>, Sched<[]> {
4970  bits<3> Pg;
4971  bits<5> Zdn;
4972  bits<5> Zm;
4973  let Inst{31-24} = 0b00000100;
4974  let Inst{23-22} = sz8_64;
4975  let Inst{21-20} = 0b01;
4976  let Inst{19}    = wide;
4977  let Inst{18-16} = opc;
4978  let Inst{15-13} = 0b100;
4979  let Inst{12-10} = Pg;
4980  let Inst{9-5}   = Zm;
4981  let Inst{4-0}   = Zdn;
4982
4983  let Constraints = "$Zdn = $_Zdn";
4984  let DestructiveInstType = DestructiveOther;
4985  let ElementSize = zprty.ElementSize;
4986}
4987
4988multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps,
4989                                  SDPatternOperator op, string revname, bit isReverseInstr = 0> {
4990  let DestructiveInstType = DestructiveBinaryCommWithRev in {
4991  def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>,
4992           SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>;
4993  def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>,
4994           SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>;
4995  def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>,
4996           SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>;
4997  def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>,
4998           SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>;
4999  }
5000  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
5001  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
5002  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
5003  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
5004}
5005
5006multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> {
5007  def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>;
5008  def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>;
5009  def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>;
5010  def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>;
5011
5012  def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>;
5013  def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>;
5014  def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>;
5015  def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>;
5016}
5017
5018multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm,
5019                                  SDPatternOperator op> {
5020  def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>;
5021  def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>;
5022  def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>;
5023
5024  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>;
5025  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>;
5026  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>;
5027}
5028
5029//===----------------------------------------------------------------------===//
5030// SVE Shift - Unpredicated Group
5031//===----------------------------------------------------------------------===//
5032
5033class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm,
5034                               ZPRRegOp zprty>
5035: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm),
5036  asm, "\t$Zd, $Zn, $Zm",
5037  "",
5038  []>, Sched<[]> {
5039  bits<5> Zd;
5040  bits<5> Zm;
5041  bits<5> Zn;
5042  let Inst{31-24} = 0b00000100;
5043  let Inst{23-22} = sz8_64;
5044  let Inst{21}    = 0b1;
5045  let Inst{20-16} = Zm;
5046  let Inst{15-12} = 0b1000;
5047  let Inst{11-10} = opc;
5048  let Inst{9-5}   = Zn;
5049  let Inst{4-0}   = Zd;
5050}
5051
5052multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm> {
5053  def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>;
5054  def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>;
5055  def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>;
5056}
5057
5058class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm,
5059                               ZPRRegOp zprty, Operand immtype>
5060: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm),
5061  asm, "\t$Zd, $Zn, $imm",
5062  "",
5063  []>, Sched<[]> {
5064  bits<5> Zd;
5065  bits<5> Zn;
5066  bits<6> imm;
5067  let Inst{31-24} = 0b00000100;
5068  let Inst{23-22} = tsz8_64{3-2};
5069  let Inst{21}    = 0b1;
5070  let Inst{20-19} = tsz8_64{1-0};
5071  let Inst{18-16} = imm{2-0}; // imm3
5072  let Inst{15-12} = 0b1001;
5073  let Inst{11-10} = opc;
5074  let Inst{9-5}   = Zn;
5075  let Inst{4-0}   = Zd;
5076}
5077
5078multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm,
5079                                           SDPatternOperator op> {
5080  def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>;
5081  def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> {
5082    let Inst{19} = imm{3};
5083  }
5084  def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> {
5085    let Inst{20-19} = imm{4-3};
5086  }
5087  def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> {
5088    let Inst{22}    = imm{5};
5089    let Inst{20-19} = imm{4-3};
5090  }
5091
5092  def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8,  !cast<Instruction>(NAME # _B)>;
5093  def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1,  i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>;
5094  def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1,  i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>;
5095  def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1,  i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>;
5096}
5097
5098multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm,
5099                                            SDPatternOperator op> {
5100  def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>;
5101  def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> {
5102    let Inst{19} = imm{3};
5103  }
5104  def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> {
5105    let Inst{20-19} = imm{4-3};
5106  }
5107  def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> {
5108    let Inst{22}    = imm{5};
5109    let Inst{20-19} = imm{4-3};
5110  }
5111
5112  def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8,  !cast<Instruction>(NAME # _B)>;
5113  def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1,  i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>;
5114  def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1,  i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>;
5115  def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1,  i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>;
5116}
5117
5118//===----------------------------------------------------------------------===//
5119// SVE Memory - Store Group
5120//===----------------------------------------------------------------------===//
5121
5122class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm,
5123                     RegisterOperand VecList>
5124: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
5125  asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
5126  "",
5127  []>, Sched<[]> {
5128  bits<3> Pg;
5129  bits<5> Rn;
5130  bits<5> Zt;
5131  bits<4> imm4;
5132  let Inst{31-25} = 0b1110010;
5133  let Inst{24-23} = msz;
5134  let Inst{22-21} = esz;
5135  let Inst{20}    = 0;
5136  let Inst{19-16} = imm4;
5137  let Inst{15-13} = 0b111;
5138  let Inst{12-10} = Pg;
5139  let Inst{9-5}   = Rn;
5140  let Inst{4-0}   = Zt;
5141
5142  let mayStore = 1;
5143}
5144
5145multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm,
5146                          RegisterOperand listty, ZPRRegOp zprty>
5147{
5148  def NAME : sve_mem_cst_si<msz, esz, asm, listty>;
5149
5150  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
5151                 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;
5152  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
5153                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
5154  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
5155                  (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
5156}
5157
5158class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
5159                     string asm, Operand immtype>
5160: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
5161  asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
5162  "",
5163  []>, Sched<[]> {
5164  bits<3> Pg;
5165  bits<5> Rn;
5166  bits<5> Zt;
5167  bits<4> imm4;
5168  let Inst{31-25} = 0b1110010;
5169  let Inst{24-23} = sz;
5170  let Inst{22-21} = nregs;
5171  let Inst{20}    = 1;
5172  let Inst{19-16} = imm4;
5173  let Inst{15-13} = 0b111;
5174  let Inst{12-10} = Pg;
5175  let Inst{9-5}   = Rn;
5176  let Inst{4-0}   = Zt;
5177
5178  let mayStore = 1;
5179}
5180
5181multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
5182                          string asm, Operand immtype> {
5183  def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>;
5184
5185  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
5186                  (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
5187}
5188
5189class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
5190                     string asm, RegisterOperand gprty>
5191: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
5192  asm, "\t$Zt, $Pg, [$Rn, $Rm]",
5193  "",
5194  []>, Sched<[]> {
5195  bits<3> Pg;
5196  bits<5> Rm;
5197  bits<5> Rn;
5198  bits<5> Zt;
5199  let Inst{31-25} = 0b1110010;
5200  let Inst{24-23} = sz;
5201  let Inst{22-21} = nregs;
5202  let Inst{20-16} = Rm;
5203  let Inst{15-13} = 0b011;
5204  let Inst{12-10} = Pg;
5205  let Inst{9-5}   = Rn;
5206  let Inst{4-0}   = Zt;
5207
5208  let mayStore = 1;
5209}
5210
5211class sve_mem_cst_ss_base<bits<4> dtype, string asm,
5212                          RegisterOperand listty, RegisterOperand gprty>
5213: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
5214  asm, "\t$Zt, $Pg, [$Rn, $Rm]",
5215  "",
5216  []>, Sched<[]> {
5217  bits<3> Pg;
5218  bits<5> Rm;
5219  bits<5> Rn;
5220  bits<5> Zt;
5221  let Inst{31-25} = 0b1110010;
5222  let Inst{24-21} = dtype;
5223  let Inst{20-16} = Rm;
5224  let Inst{15-13} = 0b010;
5225  let Inst{12-10} = Pg;
5226  let Inst{9-5}   = Rn;
5227  let Inst{4-0}   = Zt;
5228
5229  let mayStore = 1;
5230}
5231
5232multiclass sve_mem_cst_ss<bits<4> dtype, string asm,
5233                          RegisterOperand listty, ZPRRegOp zprty,
5234                          RegisterOperand gprty> {
5235  def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>;
5236
5237  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]",
5238                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
5239}
5240
5241class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList>
5242: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
5243  asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
5244  "",
5245  []>, Sched<[]> {
5246  bits<3> Pg;
5247  bits<5> Rn;
5248  bits<5> Zt;
5249  bits<4> imm4;
5250  let Inst{31-25} = 0b1110010;
5251  let Inst{24-23} = msz;
5252  let Inst{22-20} = 0b001;
5253  let Inst{19-16} = imm4;
5254  let Inst{15-13} = 0b111;
5255  let Inst{12-10} = Pg;
5256  let Inst{9-5}   = Rn;
5257  let Inst{4-0}   = Zt;
5258
5259  let mayStore = 1;
5260}
5261
5262multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty,
5263                            ZPRRegOp zprty> {
5264  def NAME : sve_mem_cstnt_si<msz, asm, listty>;
5265
5266  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
5267                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
5268  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]",
5269                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;
5270  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]",
5271                  (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
5272}
5273
5274class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty,
5275                            RegisterOperand gprty>
5276: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
5277  asm, "\t$Zt, $Pg, [$Rn, $Rm]",
5278  "",
5279  []>, Sched<[]> {
5280  bits<3> Pg;
5281  bits<5> Rm;
5282  bits<5> Rn;
5283  bits<5> Zt;
5284  let Inst{31-25} = 0b1110010;
5285  let Inst{24-23} = msz;
5286  let Inst{22-21} = 0b00;
5287  let Inst{20-16} = Rm;
5288  let Inst{15-13} = 0b011;
5289  let Inst{12-10} = Pg;
5290  let Inst{9-5}   = Rn;
5291  let Inst{4-0}   = Zt;
5292
5293  let mayStore = 1;
5294}
5295
5296multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty,
5297                            ZPRRegOp zprty, RegisterOperand gprty> {
5298  def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>;
5299
5300  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]",
5301                 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
5302}
5303
5304class sve2_mem_sstnt_vs_base<bits<3> opc, string asm,
5305                             RegisterOperand listty, ZPRRegOp zprty>
5306: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm),
5307  asm, "\t$Zt, $Pg, [$Zn, $Rm]",
5308  "",
5309  []>, Sched<[]> {
5310  bits<3> Pg;
5311  bits<5> Rm;
5312  bits<5> Zn;
5313  bits<5> Zt;
5314  let Inst{31-25} = 0b1110010;
5315  let Inst{24-22} = opc;
5316  let Inst{21}    = 0b0;
5317  let Inst{20-16} = Rm;
5318  let Inst{15-13} = 0b001;
5319  let Inst{12-10} = Pg;
5320  let Inst{9-5}   = Zn;
5321  let Inst{4-0}   = Zt;
5322
5323  let mayStore = 1;
5324}
5325
5326multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm,
5327                             SDPatternOperator op,
5328                             ValueType vt> {
5329  def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>;
5330
5331  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]",
5332                 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>;
5333  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5334                 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>;
5335  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5336                 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>;
5337
5338  def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt),
5339             (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>;
5340}
5341
5342multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm,
5343                             SDPatternOperator op,
5344                             ValueType vt> {
5345  def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>;
5346
5347  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]",
5348                 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>;
5349  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5350                 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>;
5351  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5352                 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;
5353
5354  def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt),
5355             (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>;
5356}
5357
5358class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm,
5359                     RegisterOperand VecList, RegisterOperand zprext>
5360: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
5361  asm, "\t$Zt, $Pg, [$Rn, $Zm]",
5362  "",
5363  []>, Sched<[]> {
5364  bits<3> Pg;
5365  bits<5> Rn;
5366  bits<5> Zm;
5367  bits<5> Zt;
5368  let Inst{31-25} = 0b1110010;
5369  let Inst{24-22} = opc;
5370  let Inst{21}    = scaled;
5371  let Inst{20-16} = Zm;
5372  let Inst{15}    = 0b1;
5373  let Inst{14}    = xs;
5374  let Inst{13}    = 0;
5375  let Inst{12-10} = Pg;
5376  let Inst{9-5}   = Rn;
5377  let Inst{4-0}   = Zt;
5378
5379  let mayStore = 1;
5380}
5381
5382multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm,
5383                                    SDPatternOperator sxtw_op,
5384                                    SDPatternOperator uxtw_op,
5385                                    RegisterOperand sxtw_opnd,
5386                                    RegisterOperand uxtw_opnd,
5387                                    ValueType vt > {
5388  def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>;
5389  def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>;
5390
5391  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5392                 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
5393  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5394                 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
5395
5396  def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),
5397            (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5398  def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),
5399            (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5400}
5401
5402multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm,
5403                                    SDPatternOperator sxtw_op,
5404                                    SDPatternOperator uxtw_op,
5405                                    RegisterOperand sxtw_opnd,
5406                                    RegisterOperand uxtw_opnd,
5407                                    ValueType vt > {
5408  def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>;
5409  def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>;
5410
5411  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5412                 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
5413  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5414                 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
5415
5416  def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),
5417            (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5418  def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),
5419            (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5420}
5421
5422multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm,
5423                                         SDPatternOperator sxtw_op,
5424                                         SDPatternOperator uxtw_op,
5425                                         RegisterOperand sxtw_opnd,
5426                                         RegisterOperand uxtw_opnd,
5427                                         ValueType vt> {
5428  def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>;
5429  def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>;
5430
5431  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5432                 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
5433  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5434                 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
5435
5436  def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),
5437            (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5438  def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),
5439            (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5440}
5441
5442multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm,
5443                                          SDPatternOperator sxtw_op,
5444                                          SDPatternOperator uxtw_op,
5445                                          RegisterOperand sxtw_opnd,
5446                                          RegisterOperand uxtw_opnd,
5447                                          ValueType vt> {
5448  def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>;
5449  def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>;
5450
5451  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5452                 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
5453  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5454                 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
5455
5456  def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),
5457            (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5458  def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt),
5459            (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5460}
5461
5462class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm,
5463                      RegisterOperand zprext>
5464: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
5465  asm, "\t$Zt, $Pg, [$Rn, $Zm]",
5466  "",
5467  []>, Sched<[]> {
5468  bits<3> Pg;
5469  bits<5> Rn;
5470  bits<5> Zm;
5471  bits<5> Zt;
5472  let Inst{31-25} = 0b1110010;
5473  let Inst{24-23} = msz;
5474  let Inst{22}    = 0b0;
5475  let Inst{21}    = scaled;
5476  let Inst{20-16} = Zm;
5477  let Inst{15-13} = 0b101;
5478  let Inst{12-10} = Pg;
5479  let Inst{9-5}   = Rn;
5480  let Inst{4-0}   = Zt;
5481
5482  let mayStore = 1;
5483}
5484
5485multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm,
5486                                    SDPatternOperator op,
5487                                    RegisterOperand zprext,
5488                                    ValueType vt> {
5489  def _SCALED_REAL : sve_mem_sst_sv2<msz, 1, asm, zprext>;
5490
5491  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5492                 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>;
5493
5494  def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt),
5495            (!cast<Instruction>(NAME # _SCALED_REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>;
5496}
5497
5498multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm,
5499                                      SDPatternOperator op,
5500                                      ValueType vt> {
5501  def _REAL : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>;
5502
5503  def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]",
5504                 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>;
5505
5506  def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt),
5507            (!cast<Instruction>(NAME # _REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
5508}
5509
5510class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty,
5511                     RegisterOperand VecList, Operand imm_ty>
5512: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5),
5513  asm, "\t$Zt, $Pg, [$Zn, $imm5]",
5514  "",
5515  []>, Sched<[]> {
5516  bits<3> Pg;
5517  bits<5> imm5;
5518  bits<5> Zn;
5519  bits<5> Zt;
5520  let Inst{31-25} = 0b1110010;
5521  let Inst{24-23} = opc{2-1};
5522  let Inst{22}    = 0b1;
5523  let Inst{21}    = opc{0};
5524  let Inst{20-16} = imm5;
5525  let Inst{15-13} = 0b101;
5526  let Inst{12-10} = Pg;
5527  let Inst{9-5}   = Zn;
5528  let Inst{4-0}   = Zt;
5529
5530  let mayStore = 1;
5531}
5532
5533multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm,
5534                                   Operand imm_ty,
5535                                   SDPatternOperator op,
5536                                   ValueType vt> {
5537  def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>;
5538
5539  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5540                  (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>;
5541  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]",
5542                  (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>;
5543  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5544                  (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;
5545
5546  def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt),
5547            (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;
5548}
5549
5550multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm,
5551                                   Operand imm_ty,
5552                                   SDPatternOperator op,
5553                                   ValueType vt> {
5554  def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>;
5555
5556  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5557                  (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>;
5558  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]",
5559                  (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>;
5560  def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]",
5561                  (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;
5562
5563  def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt),
5564            (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;
5565}
5566
5567class sve_mem_z_spill<string asm>
5568: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9),
5569  asm, "\t$Zt, [$Rn, $imm9, mul vl]",
5570  "",
5571  []>, Sched<[]> {
5572  bits<5> Rn;
5573  bits<5> Zt;
5574  bits<9> imm9;
5575  let Inst{31-22} = 0b1110010110;
5576  let Inst{21-16} = imm9{8-3};
5577  let Inst{15-13} = 0b010;
5578  let Inst{12-10} = imm9{2-0};
5579  let Inst{9-5}   = Rn;
5580  let Inst{4-0}   = Zt;
5581
5582  let mayStore = 1;
5583}
5584
5585multiclass sve_mem_z_spill<string asm> {
5586  def NAME : sve_mem_z_spill<asm>;
5587
5588  def : InstAlias<asm # "\t$Zt, [$Rn]",
5589                  (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>;
5590}
5591
5592class sve_mem_p_spill<string asm>
5593: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9),
5594  asm, "\t$Pt, [$Rn, $imm9, mul vl]",
5595  "",
5596  []>, Sched<[]> {
5597  bits<4> Pt;
5598  bits<5> Rn;
5599  bits<9> imm9;
5600  let Inst{31-22} = 0b1110010110;
5601  let Inst{21-16} = imm9{8-3};
5602  let Inst{15-13} = 0b000;
5603  let Inst{12-10} = imm9{2-0};
5604  let Inst{9-5}   = Rn;
5605  let Inst{4}     = 0b0;
5606  let Inst{3-0}   = Pt;
5607
5608  let mayStore = 1;
5609}
5610
5611multiclass sve_mem_p_spill<string asm> {
5612  def NAME : sve_mem_p_spill<asm>;
5613
5614  def : InstAlias<asm # "\t$Pt, [$Rn]",
5615                  (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>;
5616}
5617
5618//===----------------------------------------------------------------------===//
5619// SVE Permute - Predicates Group
5620//===----------------------------------------------------------------------===//
5621
5622class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm,
5623                               PPRRegOp pprty>
5624: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm),
5625  asm, "\t$Pd, $Pn, $Pm",
5626  "", []>, Sched<[]> {
5627  bits<4> Pd;
5628  bits<4> Pm;
5629  bits<4> Pn;
5630  let Inst{31-24} = 0b00000101;
5631  let Inst{23-22} = sz8_64;
5632  let Inst{21-20} = 0b10;
5633  let Inst{19-16} = Pm;
5634  let Inst{15-13} = 0b010;
5635  let Inst{12-10} = opc;
5636  let Inst{9}     = 0b0;
5637  let Inst{8-5}   = Pn;
5638  let Inst{4}     = 0b0;
5639  let Inst{3-0}   = Pd;
5640}
5641
5642multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm,
5643                                    SDPatternOperator op> {
5644  def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>;
5645  def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>;
5646  def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>;
5647  def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>;
5648
5649  def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>;
5650  def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1,  nxv8i1,  !cast<Instruction>(NAME # _H)>;
5651  def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1,  nxv4i1,  !cast<Instruction>(NAME # _S)>;
5652  def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1,  nxv2i1,  !cast<Instruction>(NAME # _D)>;
5653}
5654
5655class sve_int_perm_punpk<bit opc, string asm>
5656: I<(outs PPR16:$Pd), (ins PPR8:$Pn),
5657  asm, "\t$Pd, $Pn",
5658  "",
5659  []>, Sched<[]> {
5660  bits<4> Pd;
5661  bits<4> Pn;
5662  let Inst{31-17} = 0b000001010011000;
5663  let Inst{16}    = opc;
5664  let Inst{15-9}  = 0b0100000;
5665  let Inst{8-5}   = Pn;
5666  let Inst{4}     = 0b0;
5667  let Inst{3-0}   = Pd;
5668}
5669
5670multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> {
5671  def NAME : sve_int_perm_punpk<opc, asm>;
5672
5673  def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>;
5674  def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1,  !cast<Instruction>(NAME)>;
5675  def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1,  !cast<Instruction>(NAME)>;
5676}
5677
5678class sve_int_rdffr_pred<bit s, string asm>
5679: I<(outs PPR8:$Pd), (ins PPRAny:$Pg),
5680  asm, "\t$Pd, $Pg/z",
5681  "",
5682  []>, Sched<[]> {
5683  bits<4> Pd;
5684  bits<4> Pg;
5685  let Inst{31-23} = 0b001001010;
5686  let Inst{22}    = s;
5687  let Inst{21-9}  = 0b0110001111000;
5688  let Inst{8-5}   = Pg;
5689  let Inst{4}     = 0;
5690  let Inst{3-0}   = Pd;
5691
5692  let Defs = !if(s, [NZCV], []);
5693  let Uses = [FFR];
5694}
5695
5696multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> {
5697  def _REAL : sve_int_rdffr_pred<s, asm>;
5698
5699  // We need a layer of indirection because early machine code passes balk at
5700  // physical register (i.e. FFR) uses that have no previous definition.
5701  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
5702  def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>,
5703           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>;
5704  }
5705}
5706
5707class sve_int_rdffr_unpred<string asm> : I<
5708  (outs PPR8:$Pd), (ins),
5709  asm, "\t$Pd",
5710  "",
5711  []>, Sched<[]> {
5712  bits<4> Pd;
5713  let Inst{31-4} = 0b0010010100011001111100000000;
5714  let Inst{3-0}   = Pd;
5715
5716  let Uses = [FFR];
5717}
5718
5719multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> {
5720  def _REAL : sve_int_rdffr_unpred<asm>;
5721
5722  // We need a layer of indirection because early machine code passes balk at
5723  // physical register (i.e. FFR) uses that have no previous definition.
5724  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
5725  def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>,
5726           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>;
5727  }
5728}
5729
5730class sve_int_wrffr<string asm, SDPatternOperator op>
5731: I<(outs), (ins PPR8:$Pn),
5732  asm, "\t$Pn",
5733  "",
5734  [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> {
5735  bits<4> Pn;
5736  let Inst{31-9} = 0b00100101001010001001000;
5737  let Inst{8-5}  = Pn;
5738  let Inst{4-0}  = 0b00000;
5739
5740  let hasSideEffects = 1;
5741  let Defs = [FFR];
5742}
5743
5744class sve_int_setffr<string asm, SDPatternOperator op>
5745: I<(outs), (ins),
5746  asm, "",
5747  "",
5748  [(op)]>, Sched<[]> {
5749  let Inst{31-0} = 0b00100101001011001001000000000000;
5750
5751  let hasSideEffects = 1;
5752  let Defs = [FFR];
5753}
5754
5755//===----------------------------------------------------------------------===//
5756// SVE Permute Vector - Predicated Group
5757//===----------------------------------------------------------------------===//
5758
5759class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm,
5760                            ZPRRegOp zprty, RegisterClass rt>
5761: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm),
5762  asm, "\t$Rdn, $Pg, $_Rdn, $Zm",
5763  "",
5764  []>, Sched<[]> {
5765  bits<3> Pg;
5766  bits<5> Rdn;
5767  bits<5> Zm;
5768  let Inst{31-24} = 0b00000101;
5769  let Inst{23-22} = sz8_64;
5770  let Inst{21-17} = 0b11000;
5771  let Inst{16}    = ab;
5772  let Inst{15-13} = 0b101;
5773  let Inst{12-10} = Pg;
5774  let Inst{9-5}   = Zm;
5775  let Inst{4-0}   = Rdn;
5776
5777  let Constraints = "$Rdn = $_Rdn";
5778}
5779
5780multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> {
5781  def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>;
5782  def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>;
5783  def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>;
5784  def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>;
5785
5786  def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>;
5787  def : SVE_3_Op_Pat<i32, op, nxv8i1,  i32, nxv8i16, !cast<Instruction>(NAME # _H)>;
5788  def : SVE_3_Op_Pat<i32, op, nxv4i1,  i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
5789  def : SVE_3_Op_Pat<i64, op, nxv2i1,  i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
5790}
5791
5792class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm,
5793                            ZPRRegOp zprty, RegisterClass rt>
5794: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm),
5795  asm, "\t$Vdn, $Pg, $_Vdn, $Zm",
5796  "",
5797  []>, Sched<[]> {
5798  bits<3> Pg;
5799  bits<5> Vdn;
5800  bits<5> Zm;
5801  let Inst{31-24} = 0b00000101;
5802  let Inst{23-22} = sz8_64;
5803  let Inst{21-17} = 0b10101;
5804  let Inst{16}    = ab;
5805  let Inst{15-13} = 0b100;
5806  let Inst{12-10} = Pg;
5807  let Inst{9-5}   = Zm;
5808  let Inst{4-0}   = Vdn;
5809
5810  let Constraints = "$Vdn = $_Vdn";
5811}
5812
5813multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> {
5814  def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>;
5815  def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>;
5816  def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>;
5817  def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>;
5818
5819  def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
5820  def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
5821  def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
5822
5823  def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
5824}
5825
5826class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm,
5827                            ZPRRegOp zprty>
5828: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
5829  asm, "\t$Zdn, $Pg, $_Zdn, $Zm",
5830  "",
5831  []>, Sched<[]> {
5832  bits<3> Pg;
5833  bits<5> Zdn;
5834  bits<5> Zm;
5835  let Inst{31-24} = 0b00000101;
5836  let Inst{23-22} = sz8_64;
5837  let Inst{21-17} = 0b10100;
5838  let Inst{16}    = ab;
5839  let Inst{15-13} = 0b100;
5840  let Inst{12-10} = Pg;
5841  let Inst{9-5}   = Zm;
5842  let Inst{4-0}   = Zdn;
5843
5844  let Constraints = "$Zdn = $_Zdn";
5845  let DestructiveInstType = DestructiveOther;
5846  let ElementSize = ElementSizeNone;
5847}
5848
5849multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> {
5850  def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>;
5851  def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>;
5852  def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>;
5853  def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>;
5854
5855  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
5856  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
5857  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
5858  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
5859
5860  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
5861  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
5862  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
5863
5864  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
5865}
5866
5867class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm,
5868                          ZPRRegOp zprty, RegisterClass resultRegType>
5869: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn),
5870  asm, "\t$Rd, $Pg, $Zn",
5871  "",
5872  []>, Sched<[]> {
5873  bits<3> Pg;
5874  bits<5> Rd;
5875  bits<5> Zn;
5876  let Inst{31-24} = 0b00000101;
5877  let Inst{23-22} = sz8_64;
5878  let Inst{21-17} = 0b10000;
5879  let Inst{16}    = ab;
5880  let Inst{15-13} = 0b101;
5881  let Inst{12-10} = Pg;
5882  let Inst{9-5}   = Zn;
5883  let Inst{4-0}   = Rd;
5884}
5885
5886multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> {
5887  def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>;
5888  def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>;
5889  def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>;
5890  def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>;
5891
5892  def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
5893  def : SVE_2_Op_Pat<i32, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
5894  def : SVE_2_Op_Pat<i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
5895  def : SVE_2_Op_Pat<i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
5896}
5897
5898class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm,
5899                          ZPRRegOp zprty, RegisterClass dstRegtype>
5900: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),
5901  asm, "\t$Vd, $Pg, $Zn",
5902  "",
5903  []>, Sched<[]> {
5904  bits<3> Pg;
5905  bits<5> Vd;
5906  bits<5> Zn;
5907  let Inst{31-24} = 0b00000101;
5908  let Inst{23-22} = sz8_64;
5909  let Inst{21-17} = 0b10001;
5910  let Inst{16}    = ab;
5911  let Inst{15-13} = 0b100;
5912  let Inst{12-10} = Pg;
5913  let Inst{9-5}   = Zn;
5914  let Inst{4-0}   = Vd;
5915}
5916
5917multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> {
5918  def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>;
5919  def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>;
5920  def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>;
5921  def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>;
5922
5923  def : SVE_2_Op_Pat<f16, op, nxv8i1,  nxv8f16, !cast<Instruction>(NAME # _H)>;
5924  def : SVE_2_Op_Pat<f32, op, nxv4i1,  nxv4f32, !cast<Instruction>(NAME # _S)>;
5925  def : SVE_2_Op_Pat<f32, op, nxv2i1,  nxv2f32, !cast<Instruction>(NAME # _S)>;
5926  def : SVE_2_Op_Pat<f64, op, nxv2i1,  nxv2f64, !cast<Instruction>(NAME # _D)>;
5927
5928  def : SVE_2_Op_Pat<bf16, op, nxv8i1,  nxv8bf16, !cast<Instruction>(NAME # _H)>;
5929}
5930
5931class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty>
5932: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm),
5933  asm, "\t$Zdn, $Pg, $_Zdn, $Zm",
5934  "",
5935  []>, Sched<[]> {
5936  bits<3> Pg;
5937  bits<5> Zdn;
5938  bits<5> Zm;
5939  let Inst{31-24} = 0b00000101;
5940  let Inst{23-22} = sz8_64;
5941  let Inst{21-13} = 0b101100100;
5942  let Inst{12-10} = Pg;
5943  let Inst{9-5}   = Zm;
5944  let Inst{4-0}   = Zdn;
5945
5946  let Constraints = "$Zdn = $_Zdn";
5947  let DestructiveInstType = DestructiveOther;
5948  let ElementSize = ElementSizeNone;
5949}
5950
5951multiclass sve_int_perm_splice<string asm, SDPatternOperator op> {
5952  def _B : sve_int_perm_splice<0b00, asm, ZPR8>;
5953  def _H : sve_int_perm_splice<0b01, asm, ZPR16>;
5954  def _S : sve_int_perm_splice<0b10, asm, ZPR32>;
5955  def _D : sve_int_perm_splice<0b11, asm, ZPR64>;
5956
5957  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
5958  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
5959  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
5960  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
5961
5962  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1,  nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>;
5963  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1,  nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>;
5964  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1,  nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>;
5965
5966  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>;
5967}
5968
5969class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm,
5970                               ZPRRegOp zprty, RegisterOperand VecList>
5971: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn),
5972  asm, "\t$Zd, $Pg, $Zn",
5973  "",
5974  []>, Sched<[]> {
5975  bits<3> Pg;
5976  bits<5> Zn;
5977  bits<5> Zd;
5978  let Inst{31-24} = 0b00000101;
5979  let Inst{23-22} = sz8_64;
5980  let Inst{21-13} = 0b101101100;
5981  let Inst{12-10} = Pg;
5982  let Inst{9-5}   = Zn;
5983  let Inst{4-0}   = Zd;
5984}
5985
5986multiclass sve2_int_perm_splice_cons<string asm> {
5987  def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8,  ZZ_b>;
5988  def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>;
5989  def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>;
5990  def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>;
5991}
5992
5993class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm,
5994                       ZPRRegOp zprty>
5995: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn),
5996  asm, "\t$Zd, $Pg/m, $Zn",
5997  "",
5998  []>, Sched<[]> {
5999  bits<5> Zd;
6000  bits<3> Pg;
6001  bits<5> Zn;
6002  let Inst{31-24} = 0b00000101;
6003  let Inst{23-22} = sz8_64;
6004  let Inst{21-18} = 0b1001;
6005  let Inst{17-16} = opc;
6006  let Inst{15-13} = 0b100;
6007  let Inst{12-10} = Pg;
6008  let Inst{9-5}   = Zn;
6009  let Inst{4-0}   = Zd;
6010
6011  let Constraints = "$Zd = $_Zd";
6012  let DestructiveInstType = DestructiveOther;
6013  let ElementSize = zprty.ElementSize;
6014}
6015
6016multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> {
6017  def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>;
6018  def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>;
6019  def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>;
6020  def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>;
6021
6022  def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
6023  def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
6024  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
6025  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
6026}
6027
6028multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> {
6029  def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>;
6030  def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>;
6031  def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>;
6032
6033  def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
6034  def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
6035  def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
6036}
6037
6038multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> {
6039  def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>;
6040  def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>;
6041
6042  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
6043  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
6044}
6045
6046multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> {
6047  def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>;
6048
6049  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
6050}
6051
6052class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty,
6053                         RegisterClass srcRegType>
6054: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn),
6055  asm, "\t$Zd, $Pg/m, $Rn",
6056  "",
6057  []>, Sched<[]> {
6058  bits<3> Pg;
6059  bits<5> Rn;
6060  bits<5> Zd;
6061  let Inst{31-24} = 0b00000101;
6062  let Inst{23-22} = sz8_64;
6063  let Inst{21-13} = 0b101000101;
6064  let Inst{12-10} = Pg;
6065  let Inst{9-5}   = Rn;
6066  let Inst{4-0}   = Zd;
6067
6068  let Constraints = "$Zd = $_Zd";
6069  let DestructiveInstType = DestructiveOther;
6070  let ElementSize = zprty.ElementSize;
6071}
6072
6073multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> {
6074  def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>;
6075  def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>;
6076  def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>;
6077  def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>;
6078
6079  def : InstAlias<"mov $Zd, $Pg/m, $Rn",
6080                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>;
6081  def : InstAlias<"mov $Zd, $Pg/m, $Rn",
6082                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>;
6083  def : InstAlias<"mov $Zd, $Pg/m, $Rn",
6084                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>;
6085  def : InstAlias<"mov $Zd, $Pg/m, $Rn",
6086                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>;
6087
6088  def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)),
6089            (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>;
6090  def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)),
6091            (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;
6092  def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)),
6093            (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;
6094  def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)),
6095            (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>;
6096}
6097
6098class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty,
6099                         RegisterClass srcRegtype>
6100: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn),
6101  asm, "\t$Zd, $Pg/m, $Vn",
6102  "",
6103  []>, Sched<[]> {
6104  bits<3> Pg;
6105  bits<5> Vn;
6106  bits<5> Zd;
6107  let Inst{31-24} = 0b00000101;
6108  let Inst{23-22} = sz8_64;
6109  let Inst{21-13} = 0b100000100;
6110  let Inst{12-10} = Pg;
6111  let Inst{9-5}   = Vn;
6112  let Inst{4-0}   = Zd;
6113
6114  let Constraints = "$Zd = $_Zd";
6115  let DestructiveInstType = DestructiveOther;
6116  let ElementSize = zprty.ElementSize;
6117}
6118
6119multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> {
6120  def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>;
6121  def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>;
6122  def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>;
6123  def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>;
6124
6125  def : InstAlias<"mov $Zd, $Pg/m, $Vn",
6126                  (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>;
6127  def : InstAlias<"mov $Zd, $Pg/m, $Vn",
6128                  (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>;
6129  def : InstAlias<"mov $Zd, $Pg/m, $Vn",
6130                  (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>;
6131  def : InstAlias<"mov $Zd, $Pg/m, $Vn",
6132                  (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>;
6133
6134  def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)),
6135            (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;
6136  def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)),
6137            (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;
6138  def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)),
6139            (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>;
6140  def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)),
6141            (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>;
6142
6143  def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)),
6144            (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>;
6145}
6146
6147class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty>
6148: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn),
6149  asm, "\t$Zd, $Pg, $Zn",
6150  "",
6151  []>, Sched<[]> {
6152  bits<3> Pg;
6153  bits<5> Zd;
6154  bits<5> Zn;
6155  let Inst{31-23} = 0b000001011;
6156  let Inst{22}    = sz;
6157  let Inst{21-13} = 0b100001100;
6158  let Inst{12-10} = Pg;
6159  let Inst{9-5}   = Zn;
6160  let Inst{4-0}   = Zd;
6161}
6162
6163multiclass sve_int_perm_compact<string asm, SDPatternOperator op> {
6164  def _S : sve_int_perm_compact<0b0, asm, ZPR32>;
6165  def _D : sve_int_perm_compact<0b1, asm, ZPR64>;
6166
6167  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
6168  def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
6169  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
6170  def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
6171}
6172
6173//===----------------------------------------------------------------------===//
6174// SVE Memory - Contiguous Load Group
6175//===----------------------------------------------------------------------===//
6176
6177class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm,
6178                          RegisterOperand VecList>
6179: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
6180  asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
6181  "",
6182  []>, Sched<[]> {
6183  bits<3> Pg;
6184  bits<5> Rn;
6185  bits<5> Zt;
6186  bits<4> imm4;
6187  let Inst{31-25} = 0b1010010;
6188  let Inst{24-21} = dtype;
6189  let Inst{20}    = nf;
6190  let Inst{19-16} = imm4;
6191  let Inst{15-13} = 0b101;
6192  let Inst{12-10} = Pg;
6193  let Inst{9-5}   = Rn;
6194  let Inst{4-0}   = Zt;
6195
6196  let mayLoad = 1;
6197  let Uses = !if(nf, [FFR], []);
6198  let Defs = !if(nf, [FFR], []);
6199}
6200
6201multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm,
6202                               RegisterOperand listty, ZPRRegOp zprty> {
6203  def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>;
6204
6205  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6206                  (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
6207  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
6208                  (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;
6209  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6210                  (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
6211
6212  // We need a layer of indirection because early machine code passes balk at
6213  // physical register (i.e. FFR) uses that have no previous definition.
6214  let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in {
6215  def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>,
6216           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>;
6217  }
6218}
6219
6220multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty,
6221                          ZPRRegOp zprty>
6222: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>;
6223
6224class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList>
6225: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4),
6226  asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
6227  "",
6228  []>, Sched<[]> {
6229  bits<5> Zt;
6230  bits<3> Pg;
6231  bits<5> Rn;
6232  bits<4> imm4;
6233  let Inst{31-25} = 0b1010010;
6234  let Inst{24-23} = msz;
6235  let Inst{22-20} = 0b000;
6236  let Inst{19-16} = imm4;
6237  let Inst{15-13} = 0b111;
6238  let Inst{12-10} = Pg;
6239  let Inst{9-5}   = Rn;
6240  let Inst{4-0}   = Zt;
6241
6242  let mayLoad = 1;
6243}
6244
6245multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty,
6246                            ZPRRegOp zprty> {
6247  def NAME : sve_mem_cldnt_si_base<msz, asm, listty>;
6248
6249  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6250                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
6251  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
6252                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>;
6253  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6254                  (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
6255}
6256
6257class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList,
6258                            RegisterOperand gprty>
6259: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
6260  asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",
6261  "",
6262  []>, Sched<[]> {
6263  bits<3> Pg;
6264  bits<5> Rm;
6265  bits<5> Rn;
6266  bits<5> Zt;
6267  let Inst{31-25} = 0b1010010;
6268  let Inst{24-23} = msz;
6269  let Inst{22-21} = 0b00;
6270  let Inst{20-16} = Rm;
6271  let Inst{15-13} = 0b110;
6272  let Inst{12-10} = Pg;
6273  let Inst{9-5}   = Rn;
6274  let Inst{4-0}   = Zt;
6275
6276  let mayLoad = 1;
6277}
6278
6279multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty,
6280                            ZPRRegOp zprty, RegisterOperand gprty> {
6281  def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>;
6282
6283  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
6284                 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
6285}
6286
6287class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList>
6288: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4),
6289  asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> {
6290  bits<5> Zt;
6291  bits<5> Rn;
6292  bits<3> Pg;
6293  bits<4> imm4;
6294  let Inst{31-25} = 0b1010010;
6295  let Inst{24-23} = sz;
6296  let Inst{22-20} = 0;
6297  let Inst{19-16} = imm4;
6298  let Inst{15-13} = 0b001;
6299  let Inst{12-10} = Pg;
6300  let Inst{9-5}   = Rn;
6301  let Inst{4-0}   = Zt;
6302
6303  let mayLoad = 1;
6304}
6305
6306multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty,
6307                           ZPRRegOp zprty> {
6308  def NAME : sve_mem_ldqr_si<sz, asm, listty>;
6309  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6310                  (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
6311  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6312                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
6313  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]",
6314                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>;
6315}
6316
6317class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList,
6318                      RegisterOperand gprty>
6319: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
6320  asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> {
6321  bits<5> Zt;
6322  bits<3> Pg;
6323  bits<5> Rn;
6324  bits<5> Rm;
6325  let Inst{31-25} = 0b1010010;
6326  let Inst{24-23} = sz;
6327  let Inst{22-21} = 0;
6328  let Inst{20-16} = Rm;
6329  let Inst{15-13} = 0;
6330  let Inst{12-10} = Pg;
6331  let Inst{9-5}   = Rn;
6332  let Inst{4-0}   = Zt;
6333
6334  let mayLoad = 1;
6335}
6336
6337multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty,
6338                           ZPRRegOp zprty, RegisterOperand gprty> {
6339  def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>;
6340
6341  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
6342                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
6343}
6344
6345class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm,
6346                     RegisterOperand VecList, Operand immtype>
6347: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6),
6348  asm, "\t$Zt, $Pg/z, [$Rn, $imm6]",
6349  "",
6350  []>, Sched<[]> {
6351  bits<3> Pg;
6352  bits<5> Rn;
6353  bits<5> Zt;
6354  bits<6> imm6;
6355  let Inst{31-25} = 0b1000010;
6356  let Inst{24-23} = dtypeh;
6357  let Inst{22}    = 1;
6358  let Inst{21-16} = imm6;
6359  let Inst{15}    = 0b1;
6360  let Inst{14-13} = dtypel;
6361  let Inst{12-10} = Pg;
6362  let Inst{9-5}   = Rn;
6363  let Inst{4-0}   = Zt;
6364
6365  let mayLoad = 1;
6366}
6367
6368multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm,
6369                          RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> {
6370  def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>;
6371
6372  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6373                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
6374  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]",
6375                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>;
6376  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6377                  (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
6378}
6379
6380class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm,
6381                          RegisterOperand VecList>
6382: I<(outs VecList:$Zt), iops,
6383  asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",
6384  "",
6385  []>, Sched<[]> {
6386  bits<5> Zt;
6387  bits<3> Pg;
6388  bits<5> Rm;
6389  bits<5> Rn;
6390  let Inst{31-25} = 0b1010010;
6391  let Inst{24-21} = dtype;
6392  let Inst{20-16} = Rm;
6393  let Inst{15-14} = 0b01;
6394  let Inst{13}    = ff;
6395  let Inst{12-10} = Pg;
6396  let Inst{9-5}   = Rn;
6397  let Inst{4-0}   = Zt;
6398
6399  let mayLoad = 1;
6400  let Uses = !if(ff, [FFR], []);
6401  let Defs = !if(ff, [FFR], []);
6402}
6403
6404multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty,
6405                          ZPRRegOp zprty, RegisterOperand gprty> {
6406  def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
6407                               asm, listty>;
6408
6409  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
6410                 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
6411}
6412
6413multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty,
6414                            ZPRRegOp zprty, RegisterOperand gprty> {
6415  def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
6416                                  asm, listty>;
6417
6418  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
6419                 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
6420
6421  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6422                 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>;
6423
6424  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6425                 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>;
6426
6427  // We need a layer of indirection because early machine code passes balk at
6428  // physical register (i.e. FFR) uses that have no previous definition.
6429  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6430  def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>,
6431           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>;
6432  }
6433}
6434
6435multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty,
6436                            ZPRRegOp zprty>
6437: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>;
6438
6439class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
6440                     string asm, Operand immtype>
6441: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4),
6442  asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]",
6443  "",
6444  []>, Sched<[]> {
6445  bits<5> Zt;
6446  bits<3> Pg;
6447  bits<5> Rn;
6448  bits<4> imm4;
6449  let Inst{31-25} = 0b1010010;
6450  let Inst{24-23} = sz;
6451  let Inst{22-21} = nregs;
6452  let Inst{20}    = 0;
6453  let Inst{19-16} = imm4;
6454  let Inst{15-13} = 0b111;
6455  let Inst{12-10} = Pg;
6456  let Inst{9-5}   = Rn;
6457  let Inst{4-0}   = Zt;
6458
6459  let mayLoad = 1;
6460}
6461
6462multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
6463                          string asm, Operand immtype> {
6464  def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>;
6465
6466  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
6467                  (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
6468}
6469
6470class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList,
6471                     string asm, RegisterOperand gprty>
6472: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
6473  asm, "\t$Zt, $Pg/z, [$Rn, $Rm]",
6474  "",
6475  []>, Sched<[]> {
6476  bits<3> Pg;
6477  bits<5> Rm;
6478  bits<5> Rn;
6479  bits<5> Zt;
6480  let Inst{31-25} = 0b1010010;
6481  let Inst{24-23} = sz;
6482  let Inst{22-21} = nregs;
6483  let Inst{20-16} = Rm;
6484  let Inst{15-13} = 0b110;
6485  let Inst{12-10} = Pg;
6486  let Inst{9-5}   = Rn;
6487  let Inst{4-0}   = Zt;
6488
6489  let mayLoad = 1;
6490}
6491
6492//===----------------------------------------------------------------------===//
6493// SVE Memory - 32-bit Gather and Unsized Contiguous Group
6494//===----------------------------------------------------------------------===//
6495
6496// bit xs      is '1' if offsets are signed
6497// bit scaled  is '1' if the offsets are scaled
6498class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm,
6499                         RegisterOperand zprext>
6500: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
6501  asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",
6502  "",
6503  []>, Sched<[]> {
6504  bits<3> Pg;
6505  bits<5> Rn;
6506  bits<5> Zm;
6507  bits<5> Zt;
6508  let Inst{31-25} = 0b1000010;
6509  let Inst{24-23} = opc{3-2};
6510  let Inst{22}    = xs;
6511  let Inst{21}    = scaled;
6512  let Inst{20-16} = Zm;
6513  let Inst{15}    = 0b0;
6514  let Inst{14-13} = opc{1-0};
6515  let Inst{12-10} = Pg;
6516  let Inst{9-5}   = Rn;
6517  let Inst{4-0}   = Zt;
6518
6519  let mayLoad = 1;
6520  let Defs = !if(!eq(opc{0}, 1), [FFR], []);
6521  let Uses = !if(!eq(opc{0}, 1), [FFR], []);
6522}
6523
6524multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm,
6525                                        SDPatternOperator sxtw_op,
6526                                        SDPatternOperator uxtw_op,
6527                                        RegisterOperand sxtw_opnd,
6528                                        RegisterOperand uxtw_opnd,
6529                                        ValueType vt> {
6530  def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>;
6531  def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>;
6532
6533  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6534                  (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
6535  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6536                  (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
6537
6538  // We need a layer of indirection because early machine code passes balk at
6539  // physical register (i.e. FFR) uses that have no previous definition.
6540  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6541  def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>,
6542                     PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;
6543  def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>,
6544                     PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;
6545  }
6546
6547  def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)),
6548            (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;
6549  def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)),
6550            (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;
6551}
6552
6553multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm,
6554                                          SDPatternOperator sxtw_op,
6555                                          SDPatternOperator uxtw_op,
6556                                          RegisterOperand sxtw_opnd,
6557                                          RegisterOperand uxtw_opnd,
6558                                          ValueType vt> {
6559  def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>;
6560  def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>;
6561
6562  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6563                  (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
6564  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6565                  (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
6566
6567  // We need a layer of indirection because early machine code passes balk at
6568  // physical register (i.e. FFR) uses that have no previous definition.
6569  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6570  def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>,
6571              PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;
6572  def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>,
6573              PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;
6574  }
6575
6576  def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)),
6577            (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
6578  def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)),
6579            (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
6580}
6581
6582
6583class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty>
6584: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5),
6585  asm, "\t$Zt, $Pg/z, [$Zn, $imm5]",
6586  "",
6587  []>, Sched<[]> {
6588  bits<3> Pg;
6589  bits<5> Zn;
6590  bits<5> Zt;
6591  bits<5> imm5;
6592  let Inst{31-25} = 0b1000010;
6593  let Inst{24-23} = opc{3-2};
6594  let Inst{22-21} = 0b01;
6595  let Inst{20-16} = imm5;
6596  let Inst{15}    = 0b1;
6597  let Inst{14-13} = opc{1-0};
6598  let Inst{12-10} = Pg;
6599  let Inst{9-5}   = Zn;
6600  let Inst{4-0}   = Zt;
6601
6602  let mayLoad = 1;
6603  let Defs = !if(!eq(opc{0}, 1), [FFR], []);
6604  let Uses = !if(!eq(opc{0}, 1), [FFR], []);
6605}
6606
6607multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty,
6608                                      SDPatternOperator op, ValueType vt> {
6609  def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>;
6610
6611  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
6612                  (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>;
6613  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]",
6614                  (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>;
6615  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
6616                  (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;
6617
6618  // We need a layer of indirection because early machine code passes balk at
6619  // physical register (i.e. FFR) uses that have no previous definition.
6620  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6621  def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>,
6622             PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>;
6623  }
6624
6625  def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)),
6626            (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;
6627}
6628
6629class sve_mem_prfm_si<bits<2> msz, string asm>
6630: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6),
6631  asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]",
6632  "",
6633  []>, Sched<[]> {
6634  bits<5> Rn;
6635  bits<3> Pg;
6636  bits<6> imm6;
6637  bits<4> prfop;
6638  let Inst{31-22} = 0b1000010111;
6639  let Inst{21-16} = imm6;
6640  let Inst{15}    = 0b0;
6641  let Inst{14-13} = msz;
6642  let Inst{12-10} = Pg;
6643  let Inst{9-5}   = Rn;
6644  let Inst{4}     = 0b0;
6645  let Inst{3-0}   = prfop;
6646
6647  let hasSideEffects = 1;
6648}
6649
6650multiclass sve_mem_prfm_si<bits<2> msz, string asm> {
6651  def NAME : sve_mem_prfm_si<msz, asm>;
6652
6653  def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]",
6654                  (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
6655}
6656
6657class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty>
6658: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
6659  asm, "\t$prfop, $Pg, [$Rn, $Rm]",
6660  "",
6661  []>, Sched<[]> {
6662  bits<5> Rm;
6663  bits<5> Rn;
6664  bits<3> Pg;
6665  bits<4> prfop;
6666  let Inst{31-25} = 0b1000010;
6667  let Inst{24-23} = opc{2-1};
6668  let Inst{22-21} = 0b00;
6669  let Inst{20-16} = Rm;
6670  let Inst{15}    = 0b1;
6671  let Inst{14}    = opc{0};
6672  let Inst{13}    = 0b0;
6673  let Inst{12-10} = Pg;
6674  let Inst{9-5}   = Rn;
6675  let Inst{4}     = 0b0;
6676  let Inst{3-0}   = prfop;
6677
6678  let hasSideEffects = 1;
6679}
6680
6681class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm,
6682                          RegisterOperand zprext>
6683: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
6684  asm, "\t$prfop, $Pg, [$Rn, $Zm]",
6685  "",
6686  []>, Sched<[]> {
6687  bits<3> Pg;
6688  bits<5> Rn;
6689  bits<5> Zm;
6690  bits<4> prfop;
6691  let Inst{31-23} = 0b100001000;
6692  let Inst{22}    = xs;
6693  let Inst{21}    = 0b1;
6694  let Inst{20-16} = Zm;
6695  let Inst{15}    = 0b0;
6696  let Inst{14-13} = msz;
6697  let Inst{12-10} = Pg;
6698  let Inst{9-5}   = Rn;
6699  let Inst{4}     = 0b0;
6700  let Inst{3-0}   = prfop;
6701
6702  let hasSideEffects = 1;
6703}
6704
6705multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm,
6706                                      RegisterOperand sxtw_opnd,
6707                                      RegisterOperand uxtw_opnd,
6708                                      PatFrag op_sxtw,
6709                                      PatFrag op_uxtw> {
6710  def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>;
6711  def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>;
6712
6713  def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),
6714            (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;
6715
6716  def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),
6717            (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;
6718}
6719
6720class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty>
6721: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5),
6722  asm, "\t$prfop, $Pg, [$Zn, $imm5]",
6723  "",
6724  []>, Sched<[]> {
6725  bits<3> Pg;
6726  bits<5> Zn;
6727  bits<5> imm5;
6728  bits<4> prfop;
6729  let Inst{31-25} = 0b1000010;
6730  let Inst{24-23} = msz;
6731  let Inst{22-21} = 0b00;
6732  let Inst{20-16} = imm5;
6733  let Inst{15-13} = 0b111;
6734  let Inst{12-10} = Pg;
6735  let Inst{9-5}   = Zn;
6736  let Inst{4}     = 0b0;
6737  let Inst{3-0}   = prfop;
6738}
6739
6740multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> {
6741  def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>;
6742
6743  def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]",
6744                  (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>;
6745
6746  def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)),
6747            (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>;
6748}
6749
6750class sve_mem_z_fill<string asm>
6751: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9),
6752  asm, "\t$Zt, [$Rn, $imm9, mul vl]",
6753  "",
6754  []>, Sched<[]> {
6755  bits<5> Rn;
6756  bits<5> Zt;
6757  bits<9> imm9;
6758  let Inst{31-22} = 0b1000010110;
6759  let Inst{21-16} = imm9{8-3};
6760  let Inst{15-13} = 0b010;
6761  let Inst{12-10} = imm9{2-0};
6762  let Inst{9-5}   = Rn;
6763  let Inst{4-0}   = Zt;
6764
6765  let mayLoad = 1;
6766}
6767
6768multiclass sve_mem_z_fill<string asm> {
6769  def NAME : sve_mem_z_fill<asm>;
6770
6771  def : InstAlias<asm # "\t$Zt, [$Rn]",
6772                  (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>;
6773}
6774
6775class sve_mem_p_fill<string asm>
6776: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9),
6777  asm, "\t$Pt, [$Rn, $imm9, mul vl]",
6778  "",
6779  []>, Sched<[]> {
6780  bits<4> Pt;
6781  bits<5> Rn;
6782  bits<9> imm9;
6783  let Inst{31-22} = 0b1000010110;
6784  let Inst{21-16} = imm9{8-3};
6785  let Inst{15-13} = 0b000;
6786  let Inst{12-10} = imm9{2-0};
6787  let Inst{9-5}   = Rn;
6788  let Inst{4}     = 0b0;
6789  let Inst{3-0}   = Pt;
6790
6791  let mayLoad = 1;
6792}
6793
6794multiclass sve_mem_p_fill<string asm> {
6795  def NAME : sve_mem_p_fill<asm>;
6796
6797  def : InstAlias<asm # "\t$Pt, [$Rn]",
6798                  (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>;
6799}
6800
6801class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm,
6802                             RegisterOperand VecList>
6803: I<(outs VecList:$Zt), iops,
6804  asm, "\t$Zt, $Pg/z, [$Zn, $Rm]",
6805  "",
6806  []>, Sched<[]> {
6807  bits<3> Pg;
6808  bits<5> Rm;
6809  bits<5> Zn;
6810  bits<5> Zt;
6811  let Inst{31}    = 0b1;
6812  let Inst{30}    = opc{4};
6813  let Inst{29-25} = 0b00010;
6814  let Inst{24-23} = opc{3-2};
6815  let Inst{22-21} = 0b00;
6816  let Inst{20-16} = Rm;
6817  let Inst{15}    = 0b1;
6818  let Inst{14-13} = opc{1-0};
6819  let Inst{12-10} = Pg;
6820  let Inst{9-5}   = Zn;
6821  let Inst{4-0}   = Zt;
6822
6823  let mayLoad = 1;
6824}
6825
6826multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm,
6827                                  SDPatternOperator op,
6828                                  ValueType vt> {
6829  def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm),
6830                                     asm, Z_s>;
6831
6832  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]",
6833                 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>;
6834  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
6835                 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>;
6836  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
6837                 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>;
6838
6839  def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)),
6840             (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>;
6841}
6842
6843multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm,
6844                                   SDPatternOperator op,
6845                                   ValueType vt> {
6846  def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm),
6847                                     asm, Z_d>;
6848
6849  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]",
6850                 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>;
6851  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
6852                 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>;
6853  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
6854                 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>;
6855
6856  def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)),
6857             (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>;
6858}
6859
6860//===----------------------------------------------------------------------===//
6861// SVE Memory - 64-bit Gather Group
6862//===----------------------------------------------------------------------===//
6863
6864// bit xs      is '1' if offsets are signed
6865// bit scaled  is '1' if the offsets are scaled
6866// bit lsl     is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl)
6867class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm,
6868                         RegisterOperand zprext>
6869: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
6870  asm, "\t$Zt, $Pg/z, [$Rn, $Zm]",
6871  "",
6872  []>, Sched<[]> {
6873  bits<3> Pg;
6874  bits<5> Rn;
6875  bits<5> Zm;
6876  bits<5> Zt;
6877  let Inst{31-25} = 0b1100010;
6878  let Inst{24-23} = opc{3-2};
6879  let Inst{22}    = xs;
6880  let Inst{21}    = scaled;
6881  let Inst{20-16} = Zm;
6882  let Inst{15}    = lsl;
6883  let Inst{14-13} = opc{1-0};
6884  let Inst{12-10} = Pg;
6885  let Inst{9-5}   = Rn;
6886  let Inst{4-0}   = Zt;
6887
6888  let mayLoad = 1;
6889  let Defs = !if(!eq(opc{0}, 1), [FFR], []);
6890  let Uses = !if(!eq(opc{0}, 1), [FFR], []);
6891}
6892
6893multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm,
6894                                        SDPatternOperator sxtw_op,
6895                                        SDPatternOperator uxtw_op,
6896                                        RegisterOperand sxtw_opnd,
6897                                        RegisterOperand uxtw_opnd,
6898                                        ValueType vt> {
6899  def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>;
6900  def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>;
6901
6902  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6903                  (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
6904  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6905                  (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
6906
6907  // We need a layer of indirection because early machine code passes balk at
6908  // physical register (i.e. FFR) uses that have no previous definition.
6909  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6910  def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>,
6911                     PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;
6912  def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>,
6913                     PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;
6914  }
6915
6916  def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)),
6917            (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;
6918  def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)),
6919            (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;
6920}
6921
6922multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm,
6923                                          SDPatternOperator sxtw_op,
6924                                          SDPatternOperator uxtw_op,
6925                                          RegisterOperand sxtw_opnd,
6926                                          RegisterOperand uxtw_opnd,
6927                                          ValueType vt> {
6928  def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>;
6929  def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>;
6930
6931  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6932                  (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>;
6933  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6934                  (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>;
6935
6936  // We need a layer of indirection because early machine code passes balk at
6937  // physical register (i.e. FFR) uses that have no previous definition.
6938  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6939  def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>,
6940              PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;
6941  def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>,
6942              PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;
6943  }
6944
6945  def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)),
6946            (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
6947  def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)),
6948            (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
6949}
6950
6951multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm,
6952                                         SDPatternOperator op,
6953                                         RegisterOperand zprext, ValueType vt> {
6954  def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>;
6955
6956  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6957                  (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>;
6958
6959  // We need a layer of indirection because early machine code passes balk at
6960  // physical register (i.e. FFR) uses that have no previous definition.
6961  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6962  def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>,
6963                PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>;
6964  }
6965
6966  def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)),
6967                     (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>;
6968}
6969
6970multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm,
6971                                           SDPatternOperator op, ValueType vt> {
6972  def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>;
6973
6974  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]",
6975                  (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>;
6976
6977  // We need a layer of indirection because early machine code passes balk at
6978  // physical register (i.e. FFR) uses that have no previous definition.
6979  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
6980  def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>,
6981           PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>;
6982  }
6983
6984  def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)),
6985            (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>;
6986}
6987
6988class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty>
6989: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5),
6990  asm, "\t$Zt, $Pg/z, [$Zn, $imm5]",
6991  "",
6992  []>, Sched<[]> {
6993  bits<3> Pg;
6994  bits<5> Zn;
6995  bits<5> Zt;
6996  bits<5> imm5;
6997  let Inst{31-25} = 0b1100010;
6998  let Inst{24-23} = opc{3-2};
6999  let Inst{22-21} = 0b01;
7000  let Inst{20-16} = imm5;
7001  let Inst{15}    = 0b1;
7002  let Inst{14-13} = opc{1-0};
7003  let Inst{12-10} = Pg;
7004  let Inst{9-5}   = Zn;
7005  let Inst{4-0}   = Zt;
7006
7007  let mayLoad = 1;
7008  let Defs = !if(!eq(opc{0}, 1), [FFR], []);
7009  let Uses = !if(!eq(opc{0}, 1), [FFR], []);
7010}
7011
7012multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty,
7013                                      SDPatternOperator op, ValueType vt> {
7014  def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>;
7015
7016  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
7017                  (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>;
7018  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]",
7019                 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>;
7020  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]",
7021                  (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;
7022
7023  // We need a layer of indirection because early machine code passes balk at
7024  // physical register (i.e. FFR) uses that have no previous definition.
7025  let hasSideEffects = 1, hasNoSchedulingInfo = 1 in {
7026  def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>,
7027                  PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>;
7028  }
7029
7030  def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)),
7031            (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>;
7032}
7033
7034// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl)
7035class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm,
7036                          RegisterOperand zprext>
7037: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm),
7038  asm, "\t$prfop, $Pg, [$Rn, $Zm]",
7039  "",
7040  []>, Sched<[]> {
7041  bits<3> Pg;
7042  bits<5> Rn;
7043  bits<5> Zm;
7044  bits<4> prfop;
7045  let Inst{31-23} = 0b110001000;
7046  let Inst{22}    = xs;
7047  let Inst{21}    = 0b1;
7048  let Inst{20-16} = Zm;
7049  let Inst{15}    = lsl;
7050  let Inst{14-13} = msz;
7051  let Inst{12-10} = Pg;
7052  let Inst{9-5}   = Rn;
7053  let Inst{4}     = 0b0;
7054  let Inst{3-0}   = prfop;
7055
7056  let hasSideEffects = 1;
7057}
7058
7059multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm,
7060                                          RegisterOperand sxtw_opnd,
7061                                          RegisterOperand uxtw_opnd,
7062                                          PatFrag op_sxtw,
7063                                          PatFrag op_uxtw> {
7064  def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>;
7065  def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>;
7066
7067  def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),
7068            (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>;
7069
7070  def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)),
7071            (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>;
7072
7073}
7074
7075multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm,
7076                                          RegisterOperand zprext, PatFrag frag> {
7077  def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>;
7078
7079  def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)),
7080            (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>;
7081
7082}
7083
7084class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty>
7085: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5),
7086  asm, "\t$prfop, $Pg, [$Zn, $imm5]",
7087  "",
7088  []>, Sched<[]> {
7089  bits<3> Pg;
7090  bits<5> Zn;
7091  bits<5> imm5;
7092  bits<4> prfop;
7093  let Inst{31-25} = 0b1100010;
7094  let Inst{24-23} = msz;
7095  let Inst{22-21} = 0b00;
7096  let Inst{20-16} = imm5;
7097  let Inst{15-13} = 0b111;
7098  let Inst{12-10} = Pg;
7099  let Inst{9-5}   = Zn;
7100  let Inst{4}     = 0b0;
7101  let Inst{3-0}   = prfop;
7102
7103  let hasSideEffects = 1;
7104}
7105
7106multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> {
7107  def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>;
7108
7109  def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]",
7110                  (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>;
7111
7112  def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)),
7113            (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>;
7114}
7115
7116//===----------------------------------------------------------------------===//
7117// SVE Compute Vector Address Group
7118//===----------------------------------------------------------------------===//
7119
7120class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm,
7121                                ZPRRegOp zprty, RegisterOperand zprext>
7122: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm),
7123  asm, "\t$Zd, [$Zn, $Zm]",
7124  "",
7125  []>, Sched<[]> {
7126  bits<5> Zd;
7127  bits<5> Zn;
7128  bits<5> Zm;
7129  let Inst{31-24} = 0b00000100;
7130  let Inst{23-22} = opc;
7131  let Inst{21}    = 0b1;
7132  let Inst{20-16} = Zm;
7133  let Inst{15-12} = 0b1010;
7134  let Inst{11-10} = msz;
7135  let Inst{9-5}   = Zn;
7136  let Inst{4-0}   = Zd;
7137}
7138
7139multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> {
7140  def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>;
7141  def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>;
7142  def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>;
7143  def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>;
7144}
7145
7146multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> {
7147  def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>;
7148  def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>;
7149  def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>;
7150  def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>;
7151}
7152
7153multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> {
7154  def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>;
7155  def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>;
7156  def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>;
7157  def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>;
7158}
7159
7160multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> {
7161  def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>;
7162  def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>;
7163  def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>;
7164  def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>;
7165}
7166
7167//===----------------------------------------------------------------------===//
7168// SVE Integer Misc - Unpredicated Group
7169//===----------------------------------------------------------------------===//
7170
7171class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty>
7172: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
7173  asm, "\t$Zd, $Zn, $Zm",
7174  "",
7175  []>, Sched<[]> {
7176  bits<5> Zd;
7177  bits<5> Zm;
7178  bits<5> Zn;
7179  let Inst{31-24} = 0b00000100;
7180  let Inst{23-22} = sz;
7181  let Inst{21}    = 0b1;
7182  let Inst{20-16} = Zm;
7183  let Inst{15-10} = 0b101100;
7184  let Inst{9-5}   = Zn;
7185  let Inst{4-0}   = Zd;
7186}
7187
7188multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> {
7189  def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>;
7190  def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>;
7191  def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>;
7192
7193  def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>;
7194  def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>;
7195  def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>;
7196}
7197
7198class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty>
7199: I<(outs zprty:$Zd), (ins zprty:$Zn),
7200  asm, "\t$Zd, $Zn",
7201  "",
7202  []>, Sched<[]> {
7203  bits<5> Zd;
7204  bits<5> Zn;
7205  let Inst{31-24} = 0b00000100;
7206  let Inst{23-22} = opc{7-6};
7207  let Inst{21}    = 0b1;
7208  let Inst{20-16} = opc{5-1};
7209  let Inst{15-11} = 0b10111;
7210  let Inst{10}    = opc{0};
7211  let Inst{9-5}   = Zn;
7212  let Inst{4-0}   = Zd;
7213}
7214
7215multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> {
7216  def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>;
7217  def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>;
7218  def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>;
7219
7220  def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>;
7221  def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>;
7222  def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>;
7223}
7224
7225//===----------------------------------------------------------------------===//
7226// SVE Integer Reduction Group
7227//===----------------------------------------------------------------------===//
7228
7229class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm,
7230                     ZPRRegOp zprty, FPRasZPROperand dstOpType>
7231: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn),
7232  asm, "\t$Vd, $Pg, $Zn",
7233  "",
7234  []>, Sched<[]> {
7235  bits<3> Pg;
7236  bits<5> Vd;
7237  bits<5> Zn;
7238  let Inst{31-24} = 0b00000100;
7239  let Inst{23-22} = sz8_32;
7240  let Inst{21}    = 0b0;
7241  let Inst{20-19} = fmt;
7242  let Inst{18-16} = opc;
7243  let Inst{15-13} = 0b001;
7244  let Inst{12-10} = Pg;
7245  let Inst{9-5}   = Zn;
7246  let Inst{4-0}   = Vd;
7247}
7248
7249multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm,
7250                                  SDPatternOperator op> {
7251  def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>;
7252  def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>;
7253  def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>;
7254
7255  def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
7256  def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
7257  def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
7258}
7259
7260multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm,
7261                                  SDPatternOperator op> {
7262  def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>;
7263  def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>;
7264  def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>;
7265  def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>;
7266
7267  def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
7268  def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
7269  def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
7270  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
7271}
7272
7273multiclass sve_int_reduce_1<bits<3> opc, string asm,
7274                            SDPatternOperator op> {
7275  def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>;
7276  def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>;
7277  def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>;
7278  def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>;
7279
7280  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
7281  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
7282  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
7283  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
7284}
7285
7286multiclass sve_int_reduce_2<bits<3> opc, string asm,
7287                            SDPatternOperator op> {
7288  def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>;
7289  def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>;
7290  def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>;
7291  def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>;
7292
7293  def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
7294  def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
7295  def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
7296  def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
7297}
7298
7299class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm,
7300                           ZPRRegOp zprty, string pg_suffix, dag iops>
7301: I<(outs zprty:$Zd), iops,
7302  asm, "\t$Zd, $Pg"#pg_suffix#", $Zn",
7303  "",
7304  []>, Sched<[]> {
7305  bits<3> Pg;
7306  bits<5> Zd;
7307  bits<5> Zn;
7308  let Inst{31-24} = 0b00000100;
7309  let Inst{23-22} = sz8_32;
7310  let Inst{21-19} = 0b010;
7311  let Inst{18-16} = opc;
7312  let Inst{15-13} = 0b001;
7313  let Inst{12-10} = Pg;
7314  let Inst{9-5}   = Zn;
7315  let Inst{4-0}   = Zd;
7316
7317  let ElementSize = zprty.ElementSize;
7318}
7319
7320multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> {
7321let Constraints = "$Zd = $_Zd" in {
7322  def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m",
7323                                (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>;
7324  def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m",
7325                                (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>;
7326  def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m",
7327                                (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>;
7328  def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m",
7329                                (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>;
7330}
7331}
7332
7333multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> {
7334  def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z",
7335                                (ins PPR3bAny:$Pg, ZPR8:$Zn)>;
7336  def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z",
7337                                (ins PPR3bAny:$Pg, ZPR16:$Zn)>;
7338  def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z",
7339                                (ins PPR3bAny:$Pg, ZPR32:$Zn)>;
7340  def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z",
7341                                (ins PPR3bAny:$Pg, ZPR64:$Zn)>;
7342}
7343
7344//===----------------------------------------------------------------------===//
7345// SVE Propagate Break Group
7346//===----------------------------------------------------------------------===//
7347
7348class sve_int_brkp<bits<2> opc, string asm>
7349: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm),
7350  asm, "\t$Pd, $Pg/z, $Pn, $Pm",
7351  "",
7352  []>, Sched<[]> {
7353  bits<4> Pd;
7354  bits<4> Pg;
7355  bits<4> Pm;
7356  bits<4> Pn;
7357  let Inst{31-24} = 0b00100101;
7358  let Inst{23}    = 0b0;
7359  let Inst{22}    = opc{1};
7360  let Inst{21-20} = 0b00;
7361  let Inst{19-16} = Pm;
7362  let Inst{15-14} = 0b11;
7363  let Inst{13-10} = Pg;
7364  let Inst{9}     = 0b0;
7365  let Inst{8-5}   = Pn;
7366  let Inst{4}     = opc{0};
7367  let Inst{3-0}   = Pd;
7368
7369  let Defs = !if(!eq (opc{1}, 1), [NZCV], []);
7370}
7371
7372multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> {
7373  def NAME : sve_int_brkp<opc, asm>;
7374
7375  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
7376}
7377
7378
7379//===----------------------------------------------------------------------===//
7380// SVE Partition Break Group
7381//===----------------------------------------------------------------------===//
7382
7383class sve_int_brkn<bit S, string asm>
7384: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm),
7385  asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm",
7386  "",
7387  []>, Sched<[]> {
7388  bits<4> Pdm;
7389  bits<4> Pg;
7390  bits<4> Pn;
7391  let Inst{31-23} = 0b001001010;
7392  let Inst{22}    = S;
7393  let Inst{21-14} = 0b01100001;
7394  let Inst{13-10} = Pg;
7395  let Inst{9}     = 0b0;
7396  let Inst{8-5}   = Pn;
7397  let Inst{4}     = 0b0;
7398  let Inst{3-0}   = Pdm;
7399
7400  let Constraints = "$Pdm = $_Pdm";
7401  let Defs = !if(S, [NZCV], []);
7402}
7403
7404multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> {
7405  def NAME : sve_int_brkn<opc, asm>;
7406
7407  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
7408}
7409
7410class sve_int_break<bits<3> opc, string asm, string suffix, dag iops>
7411: I<(outs PPR8:$Pd), iops,
7412  asm, "\t$Pd, $Pg"#suffix#", $Pn",
7413  "",
7414  []>, Sched<[]> {
7415  bits<4> Pd;
7416  bits<4> Pg;
7417  bits<4> Pn;
7418  let Inst{31-24} = 0b00100101;
7419  let Inst{23-22} = opc{2-1};
7420  let Inst{21-14} = 0b01000001;
7421  let Inst{13-10} = Pg;
7422  let Inst{9}     = 0b0;
7423  let Inst{8-5}   = Pn;
7424  let Inst{4}     = opc{0};
7425  let Inst{3-0}   = Pd;
7426
7427  let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", "");
7428  let Defs = !if(!eq (opc{1}, 1), [NZCV], []);
7429
7430}
7431
7432multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> {
7433  def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>;
7434
7435  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
7436}
7437
7438multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> {
7439  def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>;
7440
7441  def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>;
7442}
7443
7444//===----------------------------------------------------------------------===//
7445// SVE2 String Processing Group
7446//===----------------------------------------------------------------------===//
7447
7448class sve2_char_match<bit sz, bit opc, string asm,
7449                      PPRRegOp pprty, ZPRRegOp zprty>
7450: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),
7451  asm, "\t$Pd, $Pg/z, $Zn, $Zm",
7452  "",
7453  []>, Sched<[]> {
7454  bits<4> Pd;
7455  bits<3> Pg;
7456  bits<5> Zm;
7457  bits<5> Zn;
7458  let Inst{31-23} = 0b010001010;
7459  let Inst{22}    = sz;
7460  let Inst{21}    = 0b1;
7461  let Inst{20-16} = Zm;
7462  let Inst{15-13} = 0b100;
7463  let Inst{12-10} = Pg;
7464  let Inst{9-5}   = Zn;
7465  let Inst{4}     = opc;
7466  let Inst{3-0}   = Pd;
7467
7468  let Defs = [NZCV];
7469}
7470
7471multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> {
7472  def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>;
7473  def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>;
7474
7475  def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>;
7476  def : SVE_3_Op_Pat<nxv8i1,  op, nxv8i1,  nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>;
7477}
7478
7479//===----------------------------------------------------------------------===//
7480// SVE2 Histogram Computation - Segment Group
7481//===----------------------------------------------------------------------===//
7482
7483class sve2_hist_gen_segment<string asm, SDPatternOperator op>
7484: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm),
7485  asm, "\t$Zd, $Zn, $Zm",
7486  "",
7487  [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> {
7488  bits<5> Zd;
7489  bits<5> Zn;
7490  bits<5> Zm;
7491  let Inst{31-21} = 0b01000101001;
7492  let Inst{20-16} = Zm;
7493  let Inst{15-10} = 0b101000;
7494  let Inst{9-5}   = Zn;
7495  let Inst{4-0}   = Zd;
7496}
7497
7498//===----------------------------------------------------------------------===//
7499// SVE2 Histogram Computation - Vector Group
7500//===----------------------------------------------------------------------===//
7501
7502class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty>
7503: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm),
7504  asm, "\t$Zd, $Pg/z, $Zn, $Zm",
7505  "",
7506  []>, Sched<[]> {
7507  bits<5> Zd;
7508  bits<5> Zn;
7509  bits<3> Pg;
7510  bits<5> Zm;
7511  let Inst{31-23} = 0b010001011;
7512  let Inst{22}    = sz;
7513  let Inst{21}    = 0b1;
7514  let Inst{20-16} = Zm;
7515  let Inst{15-13} = 0b110;
7516  let Inst{12-10} = Pg;
7517  let Inst{9-5}   = Zn;
7518  let Inst{4-0}   = Zd;
7519}
7520
7521multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> {
7522  def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>;
7523  def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>;
7524
7525  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>;
7526  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>;
7527}
7528
7529//===----------------------------------------------------------------------===//
7530// SVE2 Crypto Extensions Group
7531//===----------------------------------------------------------------------===//
7532
7533class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty>
7534: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm),
7535  asm, "\t$Zd, $Zn, $Zm",
7536  "",
7537  []>, Sched<[]> {
7538  bits<5> Zd;
7539  bits<5> Zn;
7540  bits<5> Zm;
7541  let Inst{31-21} = 0b01000101001;
7542  let Inst{20-16} = Zm;
7543  let Inst{15-11} = 0b11110;
7544  let Inst{10}    = opc;
7545  let Inst{9-5}   = Zn;
7546  let Inst{4-0}   = Zd;
7547}
7548
7549multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty,
7550                                   SDPatternOperator op, ValueType vt> {
7551  def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>;
7552  def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;
7553}
7554
7555class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty>
7556: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm),
7557  asm, "\t$Zdn, $_Zdn, $Zm",
7558  "",
7559  []>, Sched<[]> {
7560  bits<5> Zdn;
7561  bits<5> Zm;
7562  let Inst{31-17} = 0b010001010010001;
7563  let Inst{16}    = opc{1};
7564  let Inst{15-11} = 0b11100;
7565  let Inst{10}    = opc{0};
7566  let Inst{9-5}   = Zm;
7567  let Inst{4-0}   = Zdn;
7568
7569  let Constraints = "$Zdn = $_Zdn";
7570}
7571
7572multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty,
7573                                  SDPatternOperator op, ValueType vt> {
7574  def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>;
7575  def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>;
7576}
7577
7578class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty>
7579: I<(outs zprty:$Zdn), (ins zprty:$_Zdn),
7580  asm, "\t$Zdn, $_Zdn",
7581  "",
7582  []>, Sched<[]> {
7583  bits<5> Zdn;
7584  let Inst{31-11} = 0b010001010010000011100;
7585  let Inst{10}    = opc;
7586  let Inst{9-5}   = 0b00000;
7587  let Inst{4-0}   = Zdn;
7588
7589  let Constraints = "$Zdn = $_Zdn";
7590}
7591
7592multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> {
7593  def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>;
7594  def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>;
7595}
7596
7597//===----------------------------------------------------------------------===//
7598// SVE BFloat16 Group
7599//===----------------------------------------------------------------------===//
7600
7601class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops>
7602: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> {
7603  bits<5> Zda;
7604  bits<5> Zn;
7605  let Inst{31-21} = 0b01100100011;
7606  let Inst{15-14} = opc;
7607  let Inst{13-10} = 0b0000;
7608  let Inst{9-5}   = Zn;
7609  let Inst{4-0}   = Zda;
7610
7611  let Constraints = "$Zda = $_Zda";
7612  let DestructiveInstType = DestructiveOther;
7613  let ElementSize = ElementSizeH;
7614}
7615
7616class sve_bfloat_dot<string asm>
7617: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm",
7618  (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> {
7619  bits<5> Zm;
7620  let Inst{20-16} = Zm;
7621}
7622
7623multiclass sve_bfloat_dot<string asm, SDPatternOperator op> {
7624  def NAME : sve_bfloat_dot<asm>;
7625  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
7626}
7627
7628class sve_bfloat_dot_indexed<string asm>
7629: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop",
7630  (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> {
7631  bits<2> iop;
7632  bits<3> Zm;
7633  let Inst{20-19} = iop;
7634  let Inst{18-16} = Zm;
7635}
7636
7637multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> {
7638  def NAME : sve_bfloat_dot_indexed<asm>;
7639  def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>;
7640}
7641
7642class sve_bfloat_matmul<string asm>
7643: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm),
7644  asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
7645  bits<5> Zm;
7646  bits<5> Zda;
7647  bits<5> Zn;
7648  let Inst{31-21} = 0b01100100011;
7649  let Inst{20-16} = Zm;
7650  let Inst{15-10} = 0b111001;
7651  let Inst{9-5}   = Zn;
7652  let Inst{4-0}   = Zda;
7653
7654  let Constraints = "$Zda = $_Zda";
7655  let DestructiveInstType = DestructiveOther;
7656  let ElementSize = ElementSizeH;
7657}
7658
7659multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> {
7660  def NAME : sve_bfloat_matmul<asm>;
7661  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
7662}
7663
7664class sve_bfloat_matmul_longvecl<bit BT, string asm>
7665: sve_bfloat_matmul<asm> {
7666  let Inst{23}    = 0b1;
7667  let Inst{14-13} = 0b00;
7668  let Inst{10}    = BT;
7669}
7670
7671multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> {
7672  def NAME : sve_bfloat_matmul_longvecl<BT, asm>;
7673  def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>;
7674}
7675
7676class sve_bfloat_matmul_longvecl_idx<bit BT, string asm>
7677: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop",
7678  (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> {
7679  bits<3> iop;
7680  bits<3> Zm;
7681  let Inst{23}    = 0b1;
7682  let Inst{20-19} = iop{2-1};
7683  let Inst{18-16} = Zm;
7684  let Inst{11}    = iop{0};
7685  let Inst{10}    = BT;
7686}
7687
7688multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> {
7689  def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>;
7690  def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>;
7691}
7692
7693class sve_bfloat_convert<bit N, string asm>
7694: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn),
7695  asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> {
7696  bits<5> Zd;
7697  bits<3> Pg;
7698  bits<5> Zn;
7699  let Inst{31-25} = 0b0110010;
7700  let Inst{24}    = N;
7701  let Inst{23-13} = 0b10001010101;
7702  let Inst{12-10} = Pg;
7703  let Inst{9-5}   = Zn;
7704  let Inst{4-0}   = Zd;
7705
7706  let Constraints = "$Zd = $_Zd";
7707  let DestructiveInstType = DestructiveOther;
7708  let hasSideEffects = 1;
7709  let ElementSize = ElementSizeS;
7710}
7711
7712multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> {
7713  def NAME : sve_bfloat_convert<N, asm>;
7714  def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>;
7715}
7716
7717//===----------------------------------------------------------------------===//
7718// SVE Integer Matrix Multiply Group
7719//===----------------------------------------------------------------------===//
7720
7721class sve_int_matmul<bits<2> uns, string asm>
7722: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm,
7723  "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
7724  bits<5> Zda;
7725  bits<5> Zn;
7726  bits<5> Zm;
7727  let Inst{31-24} = 0b01000101;
7728  let Inst{23-22} = uns;
7729  let Inst{21}    = 0;
7730  let Inst{20-16} = Zm;
7731  let Inst{15-10} = 0b100110;
7732  let Inst{9-5}   = Zn;
7733  let Inst{4-0}   = Zda;
7734
7735  let Constraints = "$Zda = $_Zda";
7736  let DestructiveInstType = DestructiveOther;
7737  let ElementSize = ZPR32.ElementSize;
7738}
7739
7740multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> {
7741  def NAME : sve_int_matmul<uns, asm>;
7742
7743  def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
7744}
7745
7746//===----------------------------------------------------------------------===//
7747// SVE Integer Dot Product Mixed Sign Group
7748//===----------------------------------------------------------------------===//
7749
7750class sve_int_dot_mixed<string asm>
7751: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm,
7752  "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
7753  bits<5> Zda;
7754  bits<5> Zn;
7755  bits<5> Zm;
7756  let Inst{31-21} = 0b01000100100;
7757  let Inst{20-16} = Zm;
7758  let Inst{15-10} = 0b011110;
7759  let Inst{9-5}   = Zn;
7760  let Inst{4-0}   = Zda;
7761
7762  let Constraints = "$Zda = $_Zda";
7763  let DestructiveInstType = DestructiveOther;
7764  let ElementSize = ZPR32.ElementSize;
7765}
7766
7767multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> {
7768  def NAME : sve_int_dot_mixed<asm>;
7769
7770  def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>;
7771}
7772
7773//===----------------------------------------------------------------------===//
7774// SVE Integer Dot Product Mixed Sign - Indexed Group
7775//===----------------------------------------------------------------------===//
7776
7777class sve_int_dot_mixed_indexed<bit U, string asm>
7778: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx),
7779    asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> {
7780  bits<5> Zda;
7781  bits<5> Zn;
7782  bits<3> Zm;
7783  bits<2> idx;
7784  let Inst{31-21} = 0b01000100101;
7785  let Inst{20-19} = idx;
7786  let Inst{18-16} = Zm;
7787  let Inst{15-11} = 0b00011;
7788  let Inst{10}    = U;
7789  let Inst{9-5}   = Zn;
7790  let Inst{4-0}   = Zda;
7791
7792  let Constraints = "$Zda = $_Zda";
7793  let DestructiveInstType = DestructiveOther;
7794  let ElementSize = ZPR32.ElementSize;
7795}
7796
7797multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> {
7798  def NAME : sve_int_dot_mixed_indexed<U, asm>;
7799
7800  def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;
7801}
7802
7803//===----------------------------------------------------------------------===//
7804// SVE Floating Point Matrix Multiply Accumulate Group
7805//===----------------------------------------------------------------------===//
7806
7807class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty>
7808: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm),
7809    asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> {
7810  bits<5> Zda;
7811  bits<5> Zn;
7812  bits<5> Zm;
7813  let Inst{31-23} = 0b011001001;
7814  let Inst{22}    = sz;
7815  let Inst{21}    = 1;
7816  let Inst{20-16} = Zm;
7817  let Inst{15-10} = 0b111001;
7818  let Inst{9-5}   = Zn;
7819  let Inst{4-0}   = Zda;
7820
7821  let Constraints = "$Zda = $_Zda";
7822  let DestructiveInstType = DestructiveOther;
7823  let ElementSize = zprty.ElementSize;
7824}
7825
7826multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> {
7827  def NAME : sve_fp_matrix_mla<sz, asm, zprty>;
7828
7829  def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>;
7830}
7831
7832//===----------------------------------------------------------------------===//
7833// SVE Memory - Contiguous Load And Replicate 256-bit Group
7834//===----------------------------------------------------------------------===//
7835
7836class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList>
7837: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4),
7838  asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> {
7839  bits<5> Zt;
7840  bits<5> Rn;
7841  bits<3> Pg;
7842  bits<4> imm4;
7843  let Inst{31-25} = 0b1010010;
7844  let Inst{24-23} = sz;
7845  let Inst{22-20} = 0b010;
7846  let Inst{19-16} = imm4;
7847  let Inst{15-13} = 0b001;
7848  let Inst{12-10} = Pg;
7849  let Inst{9-5}   = Rn;
7850  let Inst{4-0}   = Zt;
7851
7852  let mayLoad = 1;
7853}
7854
7855multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty,
7856                           ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> {
7857  def NAME : sve_mem_ldor_si<sz, asm, listty>;
7858  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
7859                  (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>;
7860  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]",
7861                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>;
7862  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]",
7863                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>;
7864
7865  // Base addressing mode
7866  def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)),
7867            (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>;
7868  let AddedComplexity = 2 in {
7869    // Reg + Imm addressing mode
7870    def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))),
7871              (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>;
7872  }
7873}
7874
7875class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList,
7876                      RegisterOperand gprty>
7877: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm),
7878  asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> {
7879  bits<5> Zt;
7880  bits<3> Pg;
7881  bits<5> Rn;
7882  bits<5> Rm;
7883  let Inst{31-25} = 0b1010010;
7884  let Inst{24-23} = sz;
7885  let Inst{22-21} = 0b01;
7886  let Inst{20-16} = Rm;
7887  let Inst{15-13} = 0;
7888  let Inst{12-10} = Pg;
7889  let Inst{9-5}   = Rn;
7890  let Inst{4-0}   = Zt;
7891
7892  let mayLoad = 1;
7893}
7894
7895multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty,
7896                           ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty,
7897                           ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> {
7898  def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>;
7899
7900  def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]",
7901                  (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>;
7902
7903  def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))),
7904            (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>;
7905}
7906
7907//===----------------------------------------------------------------------===//
7908// SVE Interleave 128-bit Elements Group
7909//===----------------------------------------------------------------------===//
7910
7911class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm>
7912: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm),
7913  asm, "\t$Zd, $Zn, $Zm",
7914  "",
7915  []>, Sched<[]> {
7916  bits<5> Zd;
7917  bits<5> Zm;
7918  bits<5> Zn;
7919  let Inst{31-21} = 0b00000101101;
7920  let Inst{20-16} = Zm;
7921  let Inst{15-13} = 0b000;
7922  let Inst{12-11} = opc;
7923  let Inst{10}    = P;
7924  let Inst{9-5}   = Zn;
7925  let Inst{4-0}   = Zd;
7926}
7927
7928multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> {
7929  def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>;
7930
7931  def : SVE_2_Op_Pat<nxv16i8,  op, nxv16i8,  nxv16i8,  !cast<Instruction>(NAME)>;
7932  def : SVE_2_Op_Pat<nxv8i16,  op, nxv8i16,  nxv8i16,  !cast<Instruction>(NAME)>;
7933  def : SVE_2_Op_Pat<nxv8f16,  op, nxv8f16,  nxv8f16,  !cast<Instruction>(NAME)>;
7934  def : SVE_2_Op_Pat<nxv4i32,  op, nxv4i32,  nxv4i32,  !cast<Instruction>(NAME)>;
7935  def : SVE_2_Op_Pat<nxv4f32,  op, nxv4f32,  nxv4f32,  !cast<Instruction>(NAME)>;
7936  def : SVE_2_Op_Pat<nxv2i64,  op, nxv2i64,  nxv2i64,  !cast<Instruction>(NAME)>;
7937  def : SVE_2_Op_Pat<nxv2f64,  op, nxv2f64,  nxv2f64,  !cast<Instruction>(NAME)>;
7938  def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>;
7939}
7940
7941/// Addressing modes
7942def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>;
7943def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>;
7944
7945def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>;
7946def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>;
7947def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>;
7948def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>;
7949
7950// Predicated pseudo floating point two operand instructions.
7951multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> {
7952  def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;
7953  def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
7954  def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
7955
7956  def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>;
7957  def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>;
7958  def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>;
7959  def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>;
7960  def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>;
7961  def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>;
7962}
7963
7964// Predicated pseudo integer two operand instructions.
7965multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> {
7966  def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>;
7967  def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>;
7968  def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
7969  def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
7970
7971  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>;
7972  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1,  nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>;
7973  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1,  nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>;
7974  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1,  nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>;
7975}
7976
7977// As sve_int_bin_pred but when only i32 and i64 vector types are required.
7978multiclass sve_int_bin_pred_sd<SDPatternOperator op> {
7979  def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>;
7980  def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>;
7981
7982  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>;
7983  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>;
7984}
7985
7986// Predicated pseudo integer two operand instructions. Second operand is an
7987// immediate specified by imm_[bhsd].
7988multiclass sve_int_shift_pred_bhsd<SDPatternOperator op,
7989                                   ComplexPattern imm_b, ComplexPattern imm_h,
7990                                   ComplexPattern imm_s, ComplexPattern imm_d> {
7991  def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8,  Operand<i32>, FalseLanesUndef>;
7992  def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>;
7993  def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>;
7994  def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>;
7995
7996  def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>;
7997  def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1,  i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>;
7998  def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1,  i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>;
7999  def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1,  i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>;
8000}
8001