xref: /freebsd/contrib/llvm-project/llvm/lib/Target/ARM/ARMInstrMVE.td (revision 4d3fc8b0570b29fb0d6ee9525f104d52176ff0d4)
1//===-- ARMInstrMVE.td - MVE support for ARM ---------------*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file describes the ARM MVE instruction set.
10//
11//===----------------------------------------------------------------------===//
12
13// VPT condition mask
14def vpt_mask : Operand<i32> {
15  let PrintMethod = "printVPTMask";
16  let ParserMatchClass = it_mask_asmoperand;
17  let EncoderMethod = "getVPTMaskOpValue";
18  let DecoderMethod = "DecodeVPTMaskOperand";
19}
20
21// VPT/VCMP restricted predicate for sign invariant types
22def pred_restricted_i_asmoperand : AsmOperandClass {
23  let Name = "CondCodeRestrictedI";
24  let RenderMethod = "addITCondCodeOperands";
25  let PredicateMethod = "isITCondCodeRestrictedI";
26  let ParserMethod = "parseITCondCode";
27  let DiagnosticString = "condition code for sign-independent integer "#
28                         "comparison must be EQ or NE";
29}
30
31// VPT/VCMP restricted predicate for signed types
32def pred_restricted_s_asmoperand : AsmOperandClass {
33  let Name = "CondCodeRestrictedS";
34  let RenderMethod = "addITCondCodeOperands";
35  let PredicateMethod = "isITCondCodeRestrictedS";
36  let ParserMethod = "parseITCondCode";
37  let DiagnosticString = "condition code for signed integer "#
38                         "comparison must be EQ, NE, LT, GT, LE or GE";
39}
40
41// VPT/VCMP restricted predicate for unsigned types
42def pred_restricted_u_asmoperand : AsmOperandClass {
43  let Name = "CondCodeRestrictedU";
44  let RenderMethod = "addITCondCodeOperands";
45  let PredicateMethod = "isITCondCodeRestrictedU";
46  let ParserMethod = "parseITCondCode";
47  let DiagnosticString = "condition code for unsigned integer "#
48                         "comparison must be EQ, NE, HS or HI";
49}
50
51// VPT/VCMP restricted predicate for floating point
52def pred_restricted_fp_asmoperand : AsmOperandClass {
53  let Name = "CondCodeRestrictedFP";
54  let RenderMethod = "addITCondCodeOperands";
55  let PredicateMethod = "isITCondCodeRestrictedFP";
56  let ParserMethod = "parseITCondCode";
57  let DiagnosticString = "condition code for floating-point "#
58                         "comparison must be EQ, NE, LT, GT, LE or GE";
59}
60
61class VCMPPredicateOperand : Operand<i32>;
62
63def pred_basic_i : VCMPPredicateOperand {
64  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
65  let ParserMatchClass = pred_restricted_i_asmoperand;
66  let DecoderMethod = "DecodeRestrictedIPredicateOperand";
67  let EncoderMethod = "getRestrictedCondCodeOpValue";
68}
69
70def pred_basic_u : VCMPPredicateOperand {
71  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
72  let ParserMatchClass = pred_restricted_u_asmoperand;
73  let DecoderMethod = "DecodeRestrictedUPredicateOperand";
74  let EncoderMethod = "getRestrictedCondCodeOpValue";
75}
76
77def pred_basic_s : VCMPPredicateOperand {
78  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
79  let ParserMatchClass = pred_restricted_s_asmoperand;
80  let DecoderMethod = "DecodeRestrictedSPredicateOperand";
81  let EncoderMethod = "getRestrictedCondCodeOpValue";
82}
83
84def pred_basic_fp : VCMPPredicateOperand {
85  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
86  let ParserMatchClass = pred_restricted_fp_asmoperand;
87  let DecoderMethod = "DecodeRestrictedFPPredicateOperand";
88  let EncoderMethod = "getRestrictedCondCodeOpValue";
89}
90
91// Register list operands for interleaving load/stores
92def VecList2QAsmOperand : AsmOperandClass {
93  let Name = "VecListTwoMQ";
94  let ParserMethod = "parseVectorList";
95  let RenderMethod = "addMVEVecListOperands";
96  let DiagnosticString = "operand must be a list of two consecutive "#
97                         "q-registers in range [q0,q7]";
98}
99
100def VecList2Q : RegisterOperand<MQQPR, "printMVEVectorListTwoQ"> {
101  let ParserMatchClass = VecList2QAsmOperand;
102  let PrintMethod = "printMVEVectorList<2>";
103}
104
105def VecList4QAsmOperand : AsmOperandClass {
106  let Name = "VecListFourMQ";
107  let ParserMethod = "parseVectorList";
108  let RenderMethod = "addMVEVecListOperands";
109  let DiagnosticString = "operand must be a list of four consecutive "#
110                         "q-registers in range [q0,q7]";
111}
112
113def VecList4Q : RegisterOperand<MQQQQPR, "printMVEVectorListFourQ"> {
114  let ParserMatchClass = VecList4QAsmOperand;
115  let PrintMethod = "printMVEVectorList<4>";
116}
117
118// taddrmode_imm7  := reg[r0-r7] +/- (imm7 << shift)
119class TMemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
120  let Name = "TMemImm7Shift"#shift#"Offset";
121  let PredicateMethod = "isMemImm7ShiftedOffset<"#shift#",ARM::tGPRRegClassID>";
122  let RenderMethod = "addMemImmOffsetOperands";
123}
124
125class taddrmode_imm7<int shift> : MemOperand,
126    ComplexPattern<i32, 2, "SelectTAddrModeImm7<"#shift#">", []>  {
127  let ParserMatchClass = TMemImm7ShiftOffsetAsmOperand<shift>;
128  // They are printed the same way as the T2 imm8 version
129  let PrintMethod = "printT2AddrModeImm8Operand<false>";
130  // This can also be the same as the T2 version.
131  let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
132  let DecoderMethod = "DecodeTAddrModeImm7<"#shift#">";
133  let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
134}
135
136// t2addrmode_imm7  := reg +/- (imm7)
137class MemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
138  let Name = "MemImm7Shift"#shift#"Offset";
139  let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
140                        ",ARM::GPRnopcRegClassID>";
141  let RenderMethod = "addMemImmOffsetOperands";
142}
143
144def MemImm7Shift0OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<0>;
145def MemImm7Shift1OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<1>;
146def MemImm7Shift2OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<2>;
147class T2AddrMode_Imm7<int shift> : MemOperand,
148      ComplexPattern<i32, 2, "SelectT2AddrModeImm7<"#shift#">", []> {
149  let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
150  let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 0>";
151  let ParserMatchClass =
152    !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetAsmOperand");
153  let MIOperandInfo = (ops GPRnopc:$base, i32imm:$offsimm);
154}
155
156class t2addrmode_imm7<int shift> : T2AddrMode_Imm7<shift> {
157  // They are printed the same way as the imm8 version
158  let PrintMethod = "printT2AddrModeImm8Operand<false>";
159}
160
161class MemImm7ShiftOffsetWBAsmOperand<int shift> : AsmOperandClass {
162  let Name = "MemImm7Shift"#shift#"OffsetWB";
163  let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
164                        ",ARM::rGPRRegClassID>";
165  let RenderMethod = "addMemImmOffsetOperands";
166}
167
168def MemImm7Shift0OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<0>;
169def MemImm7Shift1OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<1>;
170def MemImm7Shift2OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<2>;
171
172class t2addrmode_imm7_pre<int shift> : T2AddrMode_Imm7<shift> {
173  // They are printed the same way as the imm8 version
174  let PrintMethod = "printT2AddrModeImm8Operand<true>";
175  let ParserMatchClass =
176    !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetWBAsmOperand");
177  let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 1>";
178  let MIOperandInfo = (ops rGPR:$base, i32imm:$offsim);
179}
180
181class t2am_imm7shiftOffsetAsmOperand<int shift>
182  : AsmOperandClass { let Name = "Imm7Shift"#shift; }
183def t2am_imm7shift0OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<0>;
184def t2am_imm7shift1OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<1>;
185def t2am_imm7shift2OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<2>;
186
187class t2am_imm7_offset<int shift> : MemOperand,
188      ComplexPattern<i32, 1, "SelectT2AddrModeImm7Offset<"#shift#">",
189                     [], [SDNPWantRoot]> {
190  // They are printed the same way as the imm8 version
191  let PrintMethod = "printT2AddrModeImm8OffsetOperand";
192  let ParserMatchClass =
193    !cast<AsmOperandClass>("t2am_imm7shift"#shift#"OffsetAsmOperand");
194  let EncoderMethod = "getT2ScaledImmOpValue<7,"#shift#">";
195  let DecoderMethod = "DecodeT2Imm7<"#shift#">";
196}
197
198// Operands for gather/scatter loads of the form [Rbase, Qoffsets]
199class MemRegRQOffsetAsmOperand<int shift> : AsmOperandClass {
200  let Name = "MemRegRQS"#shift#"Offset";
201  let PredicateMethod = "isMemRegRQOffset<"#shift#">";
202  let RenderMethod = "addMemRegRQOffsetOperands";
203}
204
205def MemRegRQS0OffsetAsmOperand : MemRegRQOffsetAsmOperand<0>;
206def MemRegRQS1OffsetAsmOperand : MemRegRQOffsetAsmOperand<1>;
207def MemRegRQS2OffsetAsmOperand : MemRegRQOffsetAsmOperand<2>;
208def MemRegRQS3OffsetAsmOperand : MemRegRQOffsetAsmOperand<3>;
209
210// mve_addr_rq_shift  := reg + vreg{ << UXTW #shift}
211class mve_addr_rq_shift<int shift> : MemOperand {
212  let EncoderMethod = "getMveAddrModeRQOpValue";
213  let PrintMethod = "printMveAddrModeRQOperand<"#shift#">";
214  let ParserMatchClass =
215    !cast<AsmOperandClass>("MemRegRQS"#shift#"OffsetAsmOperand");
216  let DecoderMethod = "DecodeMveAddrModeRQ";
217  let MIOperandInfo = (ops GPRnopc:$base, MQPR:$offsreg);
218}
219
220class MemRegQOffsetAsmOperand<int shift> : AsmOperandClass {
221  let Name = "MemRegQS"#shift#"Offset";
222  let PredicateMethod = "isMemRegQOffset<"#shift#">";
223  let RenderMethod = "addMemImmOffsetOperands";
224}
225
226def MemRegQS2OffsetAsmOperand : MemRegQOffsetAsmOperand<2>;
227def MemRegQS3OffsetAsmOperand : MemRegQOffsetAsmOperand<3>;
228
229// mve_addr_q_shift  := vreg {+ #imm7s2/4}
230class mve_addr_q_shift<int shift> : MemOperand {
231  let EncoderMethod = "getMveAddrModeQOpValue<"#shift#">";
232  // Can be printed same way as other reg + imm operands
233  let PrintMethod = "printT2AddrModeImm8Operand<false>";
234  let ParserMatchClass =
235    !cast<AsmOperandClass>("MemRegQS"#shift#"OffsetAsmOperand");
236  let DecoderMethod = "DecodeMveAddrModeQ<"#shift#">";
237  let MIOperandInfo = (ops MQPR:$base, i32imm:$imm);
238}
239
240// A family of classes wrapping up information about the vector types
241// used by MVE.
242class MVEVectorVTInfo<ValueType vec, ValueType dblvec,
243                      ValueType pred, ValueType dblpred,
244                      bits<2> size, string suffixletter, bit unsigned> {
245  // The LLVM ValueType representing the vector, so we can use it in
246  // ISel patterns.
247  ValueType Vec = vec;
248
249  // The LLVM ValueType representing a vector with elements double the size
250  // of those in Vec, so we can use it in ISel patterns. It is up to the
251  // invoker of this class to ensure that this is a correct choice.
252  ValueType DblVec = dblvec;
253
254  // An LLVM ValueType representing a corresponding vector of
255  // predicate bits, for use in ISel patterns that handle an IR
256  // intrinsic describing the predicated form of the instruction.
257  ValueType Pred = pred;
258
259  // Same as Pred but for DblVec rather than Vec.
260  ValueType DblPred = dblpred;
261
262  // The most common representation of the vector element size in MVE
263  // instruction encodings: a 2-bit value V representing an (8<<V)-bit
264  // vector element.
265  bits<2> Size = size;
266
267  // For vectors explicitly mentioning a signedness of integers: 0 for
268  // signed and 1 for unsigned. For anything else, undefined.
269  bit Unsigned = unsigned;
270
271  // The number of bits in a vector element, in integer form.
272  int LaneBits = !shl(8, Size);
273
274  // The suffix used in assembly language on an instruction operating
275  // on this lane if it only cares about number of bits.
276  string BitsSuffix = !if(!eq(suffixletter, "p"),
277                          !if(!eq(unsigned, 0b0), "8", "16"),
278                          !cast<string>(LaneBits));
279
280  // The suffix used on an instruction that mentions the whole type.
281  string Suffix = suffixletter # BitsSuffix;
282
283  // The letter part of the suffix only.
284  string SuffixLetter = suffixletter;
285}
286
287// Integer vector types that don't treat signed and unsigned differently.
288def MVE_v16i8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "i", ?>;
289def MVE_v8i16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "i", ?>;
290def MVE_v4i32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v2i1, 0b10, "i", ?>;
291def MVE_v2i64 : MVEVectorVTInfo<v2i64, ?,     v2i1,  ?,    0b11, "i", ?>;
292
293// Explicitly signed and unsigned integer vectors. They map to the
294// same set of LLVM ValueTypes as above, but are represented
295// differently in assembly and instruction encodings.
296def MVE_v16s8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "s", 0b0>;
297def MVE_v8s16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "s", 0b0>;
298def MVE_v4s32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v2i1, 0b10, "s", 0b0>;
299def MVE_v2s64 : MVEVectorVTInfo<v2i64, ?,     v2i1,  ?,    0b11, "s", 0b0>;
300def MVE_v16u8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b00, "u", 0b1>;
301def MVE_v8u16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b01, "u", 0b1>;
302def MVE_v4u32 : MVEVectorVTInfo<v4i32, v2i64, v4i1,  v2i1, 0b10, "u", 0b1>;
303def MVE_v2u64 : MVEVectorVTInfo<v2i64, ?,     v2i1,  ?,    0b11, "u", 0b1>;
304
305// FP vector types.
306def MVE_v8f16 : MVEVectorVTInfo<v8f16, v4f32, v8i1,  v4i1, 0b01, "f", ?>;
307def MVE_v4f32 : MVEVectorVTInfo<v4f32, v2f64, v4i1,  v2i1, 0b10, "f", ?>;
308def MVE_v2f64 : MVEVectorVTInfo<v2f64, ?,     v2i1,  ?,    0b11, "f", ?>;
309
310// Polynomial vector types.
311def MVE_v16p8 : MVEVectorVTInfo<v16i8, v8i16, v16i1, v8i1, 0b11, "p", 0b0>;
312def MVE_v8p16 : MVEVectorVTInfo<v8i16, v4i32, v8i1,  v4i1, 0b11, "p", 0b1>;
313
314multiclass MVE_TwoOpPattern<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,
315                            dag PredOperands, Instruction Inst,
316                            SDPatternOperator IdentityVec = null_frag> {
317  // Unpredicated
318  def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
319            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
320
321  // Predicated with select
322  if !ne(VTI.Size, 0b11) then {
323    def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
324                                (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
325                                             (VTI.Vec MQPR:$Qn))),
326                                (VTI.Vec MQPR:$inactive))),
327              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
328                              ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
329                              (VTI.Vec MQPR:$inactive)))>;
330
331    // Optionally with the select folded through the op
332    def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
333                           (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
334                                             (VTI.Vec MQPR:$Qn),
335                                             (VTI.Vec IdentityVec))))),
336              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
337                              ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
338                              (VTI.Vec MQPR:$Qm)))>;
339  }
340
341  // Predicated with intrinsic
342  def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)),
343                          PredOperands,
344                          (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
345            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
346                            ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
347                            (VTI.Vec MQPR:$inactive)))>;
348}
349
350multiclass MVE_TwoOpPatternDup<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,
351                               dag PredOperands, Instruction Inst,
352                               SDPatternOperator IdentityVec = null_frag> {
353  // Unpredicated
354  def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn)))),
355            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn))>;
356
357  // Predicated with select
358  if !ne(VTI.Size, 0b11) then {
359    def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
360                                (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
361                                             (VTI.Vec (ARMvdup rGPR:$Rn)))),
362                                (VTI.Vec MQPR:$inactive))),
363              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
364                              ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
365                              (VTI.Vec MQPR:$inactive)))>;
366
367    // Optionally with the select folded through the op
368    def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
369                           (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
370                                             (ARMvdup rGPR:$Rn),
371                                             (VTI.Vec IdentityVec))))),
372              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
373                              ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
374                              (VTI.Vec MQPR:$Qm)))>;
375  }
376
377  // Predicated with intrinsic
378  def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))),
379                          PredOperands,
380                          (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
381            (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
382                            ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
383                            (VTI.Vec MQPR:$inactive)))>;
384}
385
386// --------- Start of base classes for the instructions themselves
387
388class MVE_MI<dag oops, dag iops, InstrItinClass itin, string asm,
389             string ops, string cstr, bits<2> vecsize, list<dag> pattern>
390  : Thumb2XI<oops, iops, AddrModeNone, 4, itin, !strconcat(asm, "\t", ops), cstr,
391             pattern>,
392    Requires<[HasMVEInt]> {
393  let D = MVEDomain;
394  let DecoderNamespace = "MVE";
395  let VecSize = vecsize;
396}
397
398// MVE_p is used for most predicated instructions, to add the cluster
399// of input operands that provides the VPT suffix (none, T or E) and
400// the input predicate register.
401class MVE_p<dag oops, dag iops, InstrItinClass itin, string iname,
402            string suffix, string ops, vpred_ops vpred, string cstr,
403            bits<2> vecsize, list<dag> pattern=[]>
404  : MVE_MI<oops, !con(iops, (ins vpred:$vp)), itin,
405           // If the instruction has a suffix, like vadd.f32, then the
406           // VPT predication suffix goes before the dot, so the full
407           // name has to be "vadd${vp}.f32".
408           !strconcat(iname, "${vp}",
409                      !if(!eq(suffix, ""), "", !strconcat(".", suffix))),
410           ops, !strconcat(cstr, vpred.vpred_constraint), vecsize, pattern> {
411  let Inst{31-29} = 0b111;
412  let Inst{27-26} = 0b11;
413}
414
415class MVE_f<dag oops, dag iops, InstrItinClass itin, string iname,
416            string suffix, string ops, vpred_ops vpred, string cstr,
417            bits<2> vecsize, list<dag> pattern=[]>
418  : MVE_p<oops, iops, itin, iname, suffix, ops, vpred, cstr, vecsize, pattern> {
419  let Predicates = [HasMVEFloat];
420}
421
422class MVE_MI_with_pred<dag oops, dag iops, InstrItinClass itin, string asm,
423                       string ops, string cstr, list<dag> pattern>
424  : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm, !strconcat("\t", ops), cstr,
425             pattern>,
426    Requires<[HasV8_1MMainline, HasMVEInt]> {
427  let D = MVEDomain;
428  let DecoderNamespace = "MVE";
429}
430
431class MVE_VMOV_lane_base<dag oops, dag iops, InstrItinClass itin, string asm,
432                         string suffix, string ops, string cstr,
433                         list<dag> pattern>
434  : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm,
435            !if(!eq(suffix, ""), "", "." # suffix) # "\t" # ops,
436            cstr, pattern>,
437    Requires<[HasV8_1MMainline, HasMVEInt]> {
438  let D = MVEDomain;
439  let DecoderNamespace = "MVE";
440}
441
442class MVE_ScalarShift<string iname, dag oops, dag iops, string asm, string cstr,
443            list<dag> pattern=[]>
444  : MVE_MI_with_pred<oops, iops, NoItinerary, iname, asm, cstr, pattern> {
445  let Inst{31-20} = 0b111010100101;
446  let Inst{8} = 0b1;
447  let validForTailPredication=1;
448}
449
450class MVE_ScalarShiftSingleReg<string iname, dag iops, string asm, string cstr,
451                    list<dag> pattern=[]>
452  : MVE_ScalarShift<iname, (outs rGPR:$RdaDest), iops, asm, cstr, pattern> {
453  bits<4> RdaDest;
454
455  let Inst{19-16} = RdaDest{3-0};
456}
457
458class MVE_ScalarShiftSRegImm<string iname, bits<2> op5_4>
459  : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, long_shift:$imm),
460                     "$RdaSrc, $imm", "$RdaDest = $RdaSrc",
461                     [(set rGPR:$RdaDest,
462                          (i32 (!cast<Intrinsic>("int_arm_mve_" # iname)
463                                    (i32 rGPR:$RdaSrc), (i32 imm:$imm))))]> {
464  bits<5> imm;
465
466  let Inst{15} = 0b0;
467  let Inst{14-12} = imm{4-2};
468  let Inst{11-8} = 0b1111;
469  let Inst{7-6} = imm{1-0};
470  let Inst{5-4} = op5_4{1-0};
471  let Inst{3-0} = 0b1111;
472}
473
474def MVE_SQSHL : MVE_ScalarShiftSRegImm<"sqshl", 0b11>;
475def MVE_SRSHR : MVE_ScalarShiftSRegImm<"srshr", 0b10>;
476def MVE_UQSHL : MVE_ScalarShiftSRegImm<"uqshl", 0b00>;
477def MVE_URSHR : MVE_ScalarShiftSRegImm<"urshr", 0b01>;
478
479class MVE_ScalarShiftSRegReg<string iname, bits<2> op5_4>
480  : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, rGPR:$Rm),
481                     "$RdaSrc, $Rm", "$RdaDest = $RdaSrc",
482                     [(set rGPR:$RdaDest,
483                         (i32 (!cast<Intrinsic>("int_arm_mve_" # iname)
484                                   (i32 rGPR:$RdaSrc), (i32 rGPR:$Rm))))]> {
485  bits<4> Rm;
486
487  let Inst{15-12} = Rm{3-0};
488  let Inst{11-8} = 0b1111;
489  let Inst{7-6} = 0b00;
490  let Inst{5-4} = op5_4{1-0};
491  let Inst{3-0} = 0b1101;
492
493  let Unpredictable{8-6} = 0b111;
494}
495
496def MVE_SQRSHR : MVE_ScalarShiftSRegReg<"sqrshr", 0b10>;
497def MVE_UQRSHL : MVE_ScalarShiftSRegReg<"uqrshl", 0b00>;
498
499class MVE_ScalarShiftDoubleReg<string iname, dag iops, string asm,
500                               string cstr, list<dag> pattern=[]>
501  : MVE_ScalarShift<iname, (outs tGPREven:$RdaLo, tGPROdd:$RdaHi),
502                    iops, asm, cstr, pattern> {
503  bits<4> RdaLo;
504  bits<4> RdaHi;
505
506  let Inst{19-17} = RdaLo{3-1};
507  let Inst{11-9} = RdaHi{3-1};
508
509  let hasSideEffects = 0;
510}
511
512class MVE_ScalarShiftDRegImm<string iname, bits<2> op5_4, bit op16,
513                             list<dag> pattern=[]>
514  : MVE_ScalarShiftDoubleReg<
515      iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, long_shift:$imm),
516      "$RdaLo, $RdaHi, $imm", "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
517      pattern> {
518  bits<5> imm;
519
520  let Inst{16} = op16;
521  let Inst{15} = 0b0;
522  let Inst{14-12} = imm{4-2};
523  let Inst{7-6} = imm{1-0};
524  let Inst{5-4} = op5_4{1-0};
525  let Inst{3-0} = 0b1111;
526}
527
528class MVE_ScalarShiftDRegRegBase<string iname, dag iops, string asm,
529                                 bit op5, bit op16, list<dag> pattern=[]>
530  : MVE_ScalarShiftDoubleReg<
531     iname, iops, asm, "@earlyclobber $RdaHi,@earlyclobber $RdaLo,"
532                       "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
533     pattern> {
534  bits<4> Rm;
535
536  let Inst{16} = op16;
537  let Inst{15-12} = Rm{3-0};
538  let Inst{6} = 0b0;
539  let Inst{5} = op5;
540  let Inst{4} = 0b0;
541  let Inst{3-0} = 0b1101;
542
543  // Custom decoder method because of the following overlapping encodings:
544  // ASRL and SQRSHR
545  // LSLL and UQRSHL
546  // SQRSHRL and SQRSHR
547  // UQRSHLL and UQRSHL
548  let DecoderMethod = "DecodeMVEOverlappingLongShift";
549}
550
551class MVE_ScalarShiftDRegReg<string iname, bit op5, list<dag> pattern=[]>
552  : MVE_ScalarShiftDRegRegBase<
553     iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm),
554     "$RdaLo, $RdaHi, $Rm", op5, 0b0, pattern> {
555
556  let Inst{7} = 0b0;
557}
558
559class MVE_ScalarShiftDRegRegWithSat<string iname, bit op5, list<dag> pattern=[]>
560  : MVE_ScalarShiftDRegRegBase<
561     iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm, saturateop:$sat),
562     "$RdaLo, $RdaHi, $sat, $Rm", op5, 0b1, pattern> {
563  bit sat;
564
565  let Inst{7} = sat;
566}
567
568def MVE_ASRLr   : MVE_ScalarShiftDRegReg<"asrl",    0b1,  [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
569                                        (ARMasrl tGPREven:$RdaLo_src,
570                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
571def MVE_ASRLi   : MVE_ScalarShiftDRegImm<"asrl",    0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
572                                        (ARMasrl tGPREven:$RdaLo_src,
573                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
574def MVE_LSLLr   : MVE_ScalarShiftDRegReg<"lsll",    0b0,  [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
575                                        (ARMlsll tGPREven:$RdaLo_src,
576                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
577def MVE_LSLLi   : MVE_ScalarShiftDRegImm<"lsll",    0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
578                                        (ARMlsll tGPREven:$RdaLo_src,
579                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
580def MVE_LSRL    : MVE_ScalarShiftDRegImm<"lsrl",    0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
581                                        (ARMlsrl tGPREven:$RdaLo_src,
582                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
583
584def MVE_SQRSHRL : MVE_ScalarShiftDRegRegWithSat<"sqrshrl", 0b1>;
585def MVE_SQSHLL  : MVE_ScalarShiftDRegImm<"sqshll",  0b11, 0b1>;
586def MVE_SRSHRL  : MVE_ScalarShiftDRegImm<"srshrl",  0b10, 0b1>;
587
588def MVE_UQRSHLL : MVE_ScalarShiftDRegRegWithSat<"uqrshll", 0b0>;
589def MVE_UQSHLL  : MVE_ScalarShiftDRegImm<"uqshll",  0b00, 0b1>;
590def MVE_URSHRL  : MVE_ScalarShiftDRegImm<"urshrl",  0b01, 0b1>;
591
592// start of mve_rDest instructions
593
594class MVE_rDest<dag oops, dag iops, InstrItinClass itin,
595                string iname, string suffix,
596                string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>
597// Always use vpred_n and not vpred_r: with the output register being
598// a GPR and not a vector register, there can't be any question of
599// what to put in its inactive lanes.
600  : MVE_p<oops, iops, itin, iname, suffix, ops, vpred_n, cstr, vecsize, pattern> {
601
602  let Inst{25-23} = 0b101;
603  let Inst{11-9} = 0b111;
604  let Inst{4} = 0b0;
605}
606
607class MVE_VABAV<string suffix, bit U, bits<2> size>
608  : MVE_rDest<(outs rGPR:$Rda), (ins rGPR:$Rda_src, MQPR:$Qn, MQPR:$Qm),
609              NoItinerary, "vabav", suffix, "$Rda, $Qn, $Qm", "$Rda = $Rda_src",
610              size, []> {
611  bits<4> Qm;
612  bits<4> Qn;
613  bits<4> Rda;
614
615  let Inst{28} = U;
616  let Inst{22} = 0b0;
617  let Inst{21-20} = size{1-0};
618  let Inst{19-17} = Qn{2-0};
619  let Inst{16} = 0b0;
620  let Inst{15-12} = Rda{3-0};
621  let Inst{8} = 0b1;
622  let Inst{7} = Qn{3};
623  let Inst{6} = 0b0;
624  let Inst{5} = Qm{3};
625  let Inst{3-1} = Qm{2-0};
626  let Inst{0} = 0b1;
627  let horizontalReduction = 1;
628}
629
630multiclass MVE_VABAV_m<MVEVectorVTInfo VTI> {
631  def "" : MVE_VABAV<VTI.Suffix, VTI.Unsigned, VTI.Size>;
632  defvar Inst = !cast<Instruction>(NAME);
633
634  let Predicates = [HasMVEInt] in {
635    def : Pat<(i32 (int_arm_mve_vabav
636                         (i32 VTI.Unsigned),
637                         (i32 rGPR:$Rda_src),
638                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
639              (i32 (Inst (i32 rGPR:$Rda_src),
640                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
641
642    def : Pat<(i32 (int_arm_mve_vabav_predicated
643                         (i32 VTI.Unsigned),
644                         (i32 rGPR:$Rda_src),
645                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
646                         (VTI.Pred VCCR:$mask))),
647              (i32 (Inst (i32 rGPR:$Rda_src),
648                         (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
649                         ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
650  }
651}
652
653defm MVE_VABAVs8  : MVE_VABAV_m<MVE_v16s8>;
654defm MVE_VABAVs16 : MVE_VABAV_m<MVE_v8s16>;
655defm MVE_VABAVs32 : MVE_VABAV_m<MVE_v4s32>;
656defm MVE_VABAVu8  : MVE_VABAV_m<MVE_v16u8>;
657defm MVE_VABAVu16 : MVE_VABAV_m<MVE_v8u16>;
658defm MVE_VABAVu32 : MVE_VABAV_m<MVE_v4u32>;
659
660class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
661              bit A, bit U, bits<2> size, list<dag> pattern=[]>
662  : MVE_rDest<(outs tGPREven:$Rda), iops, NoItinerary,
663              iname, suffix, "$Rda, $Qm", cstr, size, pattern> {
664  bits<3> Qm;
665  bits<4> Rda;
666
667  let Inst{28} = U;
668  let Inst{22-20} = 0b111;
669  let Inst{19-18} = size{1-0};
670  let Inst{17-16} = 0b01;
671  let Inst{15-13} = Rda{3-1};
672  let Inst{12} = 0b0;
673  let Inst{8-6} = 0b100;
674  let Inst{5} = A;
675  let Inst{3-1} = Qm{2-0};
676  let Inst{0} = 0b0;
677  let horizontalReduction = 1;
678  let validForTailPredication = 1;
679}
680
681def SDTVecReduceP : SDTypeProfile<1, 2, [    // VADDLVp
682  SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>
683]>;
684def ARMVADDVs       : SDNode<"ARMISD::VADDVs", SDTVecReduce>;
685def ARMVADDVu       : SDNode<"ARMISD::VADDVu", SDTVecReduce>;
686def ARMVADDVps      : SDNode<"ARMISD::VADDVps", SDTVecReduceP>;
687def ARMVADDVpu      : SDNode<"ARMISD::VADDVpu", SDTVecReduceP>;
688
689multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
690  def acc    : MVE_VADDV<"vaddva", VTI.Suffix,
691                         (ins tGPREven:$Rda_src, MQPR:$Qm), "$Rda = $Rda_src",
692                         0b1, VTI.Unsigned, VTI.Size>;
693  def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,
694                         (ins MQPR:$Qm), "",
695                         0b0, VTI.Unsigned, VTI.Size>;
696
697  defvar InstA = !cast<Instruction>(NAME # "acc");
698  defvar InstN = !cast<Instruction>(NAME # "no_acc");
699
700  let Predicates = [HasMVEInt] in {
701    if VTI.Unsigned then {
702      def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
703                (i32 (InstN $vec))>;
704      def : Pat<(i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
705                                                      (VTI.Vec MQPR:$vec),
706                                                      (VTI.Vec ARMimmAllZerosV))))),
707                (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
708      def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
709                (i32 (InstN $vec))>;
710      def : Pat<(i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
711                (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
712      def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
713                          (i32 tGPREven:$acc))),
714                (i32 (InstA $acc, $vec))>;
715      def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
716                                                                (VTI.Vec MQPR:$vec),
717                                                                (VTI.Vec ARMimmAllZerosV))))),
718                          (i32 tGPREven:$acc))),
719                (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
720      def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
721                          (i32 tGPREven:$acc))),
722                (i32 (InstA $acc, $vec))>;
723      def : Pat<(i32 (add (i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
724                          (i32 tGPREven:$acc))),
725                (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
726    } else {
727      def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
728                (i32 (InstN $vec))>;
729      def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
730                          (i32 tGPREven:$acc))),
731                (i32 (InstA $acc, $vec))>;
732      def : Pat<(i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
733                (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
734      def : Pat<(i32 (add (i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
735                          (i32 tGPREven:$acc))),
736                (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
737    }
738
739    def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
740                                                (i32 VTI.Unsigned),
741                                                (VTI.Pred VCCR:$pred))),
742              (i32 (InstN $vec, ARMVCCThen, $pred, zero_reg))>;
743    def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
744                                                     (i32 VTI.Unsigned),
745                                                     (VTI.Pred VCCR:$pred)),
746                        (i32 tGPREven:$acc))),
747              (i32 (InstA $acc, $vec, ARMVCCThen, $pred, zero_reg))>;
748  }
749}
750
751defm MVE_VADDVs8  : MVE_VADDV_A<MVE_v16s8>;
752defm MVE_VADDVs16 : MVE_VADDV_A<MVE_v8s16>;
753defm MVE_VADDVs32 : MVE_VADDV_A<MVE_v4s32>;
754defm MVE_VADDVu8  : MVE_VADDV_A<MVE_v16u8>;
755defm MVE_VADDVu16 : MVE_VADDV_A<MVE_v8u16>;
756defm MVE_VADDVu32 : MVE_VADDV_A<MVE_v4u32>;
757
758class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,
759               bit A, bit U, list<dag> pattern=[]>
760  : MVE_rDest<(outs tGPREven:$RdaLo, tGPROdd:$RdaHi), iops, NoItinerary, iname,
761              suffix, "$RdaLo, $RdaHi, $Qm", cstr, 0b10, pattern> {
762  bits<3> Qm;
763  bits<4> RdaLo;
764  bits<4> RdaHi;
765
766  let Inst{28} = U;
767  let Inst{22-20} = RdaHi{3-1};
768  let Inst{19-18} = 0b10;
769  let Inst{17-16} = 0b01;
770  let Inst{15-13} = RdaLo{3-1};
771  let Inst{12} = 0b0;
772  let Inst{8-6} = 0b100;
773  let Inst{5} = A;
774  let Inst{3-1} = Qm{2-0};
775  let Inst{0} = 0b0;
776  let horizontalReduction = 1;
777}
778
779def SDTVecReduceL : SDTypeProfile<2, 1, [    // VADDLV
780  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>
781]>;
782def SDTVecReduceLA : SDTypeProfile<2, 3, [    // VADDLVA
783  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
784  SDTCisVec<4>
785]>;
786def SDTVecReduceLP : SDTypeProfile<2, 2, [    // VADDLVp
787  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<2>
788]>;
789def SDTVecReduceLPA : SDTypeProfile<2, 4, [    // VADDLVAp
790  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
791  SDTCisVec<4>, SDTCisVec<5>
792]>;
793
794multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {
795  def acc    : MVE_VADDLV<"vaddlva", VTI.Suffix,
796                        (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),
797                        "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
798                        0b1, VTI.Unsigned>;
799  def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,
800                        (ins MQPR:$Qm), "",
801                        0b0, VTI.Unsigned>;
802
803  defvar InstA = !cast<Instruction>(NAME # "acc");
804  defvar InstN = !cast<Instruction>(NAME # "no_acc");
805
806  defvar letter = VTI.SuffixLetter;
807  defvar ARMVADDLV = SDNode<"ARMISD::VADDLV" # letter, SDTVecReduceL>;
808  defvar ARMVADDLVA = SDNode<"ARMISD::VADDLVA" # letter, SDTVecReduceLA>;
809  defvar ARMVADDLVp = SDNode<"ARMISD::VADDLVp" # letter, SDTVecReduceLP>;
810  defvar ARMVADDLVAp = SDNode<"ARMISD::VADDLVAp" # letter, SDTVecReduceLPA>;
811
812  let Predicates = [HasMVEInt] in {
813    def : Pat<(ARMVADDLV (v4i32 MQPR:$vec)),
814              (InstN (v4i32 MQPR:$vec))>;
815    def : Pat<(ARMVADDLVA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec)),
816              (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec))>;
817    def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),
818              (InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;
819    def : Pat<(ARMVADDLVAp tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
820                           (VTI.Pred VCCR:$pred)),
821              (InstA tGPREven:$acclo, tGPROdd:$acchi, (v4i32 MQPR:$vec),
822                     ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;
823  }
824}
825
826defm MVE_VADDLVs32 : MVE_VADDLV_A<MVE_v4s32>;
827defm MVE_VADDLVu32 : MVE_VADDLV_A<MVE_v4u32>;
828
829class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
830                     bit bit_17, bit bit_7, list<dag> pattern=[]>
831  : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm),
832              NoItinerary, iname, suffix, "$RdaSrc, $Qm",
833              "$RdaDest = $RdaSrc", !if(sz, 0b01, 0b10), pattern> {
834  bits<3> Qm;
835  bits<4> RdaDest;
836
837  let Inst{28} = sz;
838  let Inst{22-20} = 0b110;
839  let Inst{19-18} = 0b11;
840  let Inst{17} = bit_17;
841  let Inst{16} = 0b0;
842  let Inst{15-12} = RdaDest{3-0};
843  let Inst{8} = 0b1;
844  let Inst{7} = bit_7;
845  let Inst{6-5} = 0b00;
846  let Inst{3-1} = Qm{2-0};
847  let Inst{0} = 0b0;
848  let horizontalReduction = 1;
849
850  let Predicates = [HasMVEFloat];
851  let hasSideEffects = 0;
852}
853
854multiclass MVE_VMINMAXNMV_p<string iname, bit notAbs, bit isMin,
855                            MVEVectorVTInfo VTI, string intrBaseName,
856                            ValueType Scalar, RegisterClass ScalarReg> {
857  def "": MVE_VMINMAXNMV<iname, VTI.Suffix, VTI.Size{0}, notAbs, isMin>;
858  defvar Inst        = !cast<Instruction>(NAME);
859  defvar unpred_intr = !cast<Intrinsic>(intrBaseName);
860  defvar pred_intr   = !cast<Intrinsic>(intrBaseName#"_predicated");
861
862  let Predicates = [HasMVEFloat] in {
863    def : Pat<(Scalar (unpred_intr (Scalar ScalarReg:$prev),
864                                   (VTI.Vec MQPR:$vec))),
865           (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),
866                                   (VTI.Vec MQPR:$vec)),
867                              ScalarReg)>;
868    def : Pat<(Scalar (pred_intr   (Scalar ScalarReg:$prev),
869                                   (VTI.Vec MQPR:$vec),
870                                   (VTI.Pred VCCR:$pred))),
871           (COPY_TO_REGCLASS (Inst (COPY_TO_REGCLASS ScalarReg:$prev, rGPR),
872                                   (VTI.Vec MQPR:$vec),
873                                   ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg),
874                              ScalarReg)>;
875  }
876}
877
878multiclass MVE_VMINMAXNMV_fty<string iname, bit notAbs, bit isMin,
879                              string intrBase> {
880  defm f32 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v4f32, intrBase,
881                              f32, SPR>;
882  defm f16 : MVE_VMINMAXNMV_p<iname, notAbs, isMin, MVE_v8f16, intrBase,
883                              f16, HPR>;
884}
885
886defm MVE_VMINNMV : MVE_VMINMAXNMV_fty<"vminnmv",  1, 1, "int_arm_mve_minnmv">;
887defm MVE_VMAXNMV : MVE_VMINMAXNMV_fty<"vmaxnmv",  1, 0, "int_arm_mve_maxnmv">;
888defm MVE_VMINNMAV: MVE_VMINMAXNMV_fty<"vminnmav", 0, 1, "int_arm_mve_minnmav">;
889defm MVE_VMAXNMAV: MVE_VMINMAXNMV_fty<"vmaxnmav", 0, 0, "int_arm_mve_maxnmav">;
890
891class MVE_VMINMAXV<string iname, string suffix, bit U, bits<2> size,
892                 bit bit_17, bit bit_7, list<dag> pattern=[]>
893  : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm), NoItinerary,
894              iname, suffix, "$RdaSrc, $Qm", "$RdaDest = $RdaSrc", size, pattern> {
895  bits<3> Qm;
896  bits<4> RdaDest;
897
898  let Inst{28} = U;
899  let Inst{22-20} = 0b110;
900  let Inst{19-18} = size{1-0};
901  let Inst{17} = bit_17;
902  let Inst{16} = 0b0;
903  let Inst{15-12} = RdaDest{3-0};
904  let Inst{8} = 0b1;
905  let Inst{7} = bit_7;
906  let Inst{6-5} = 0b00;
907  let Inst{3-1} = Qm{2-0};
908  let Inst{0} = 0b0;
909  let horizontalReduction = 1;
910}
911
912multiclass MVE_VMINMAXV_p<string iname, bit notAbs, bit isMin,
913                          MVEVectorVTInfo VTI, string intrBaseName> {
914  def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
915                       notAbs, isMin>;
916  defvar Inst        = !cast<Instruction>(NAME);
917  defvar unpred_intr = !cast<Intrinsic>(intrBaseName);
918  defvar pred_intr   = !cast<Intrinsic>(intrBaseName#"_predicated");
919  defvar base_args   = (? (i32 rGPR:$prev), (VTI.Vec MQPR:$vec));
920  defvar args        = !if(notAbs, !con(base_args, (? (i32 VTI.Unsigned))),
921                           base_args);
922
923  let Predicates = [HasMVEInt] in {
924    def : Pat<(i32 !con(args, (unpred_intr))),
925              (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec)))>;
926    def : Pat<(i32 !con(args, (pred_intr (VTI.Pred VCCR:$pred)))),
927              (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec),
928                         ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
929  }
930}
931
932multiclass MVE_VMINMAXV_ty<string iname, bit isMin, string intrBaseName> {
933  defm s8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16s8, intrBaseName>;
934  defm s16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8s16, intrBaseName>;
935  defm s32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4s32, intrBaseName>;
936  defm u8 : MVE_VMINMAXV_p<iname, 1, isMin, MVE_v16u8, intrBaseName>;
937  defm u16: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v8u16, intrBaseName>;
938  defm u32: MVE_VMINMAXV_p<iname, 1, isMin, MVE_v4u32, intrBaseName>;
939}
940
941def SDTVecReduceR : SDTypeProfile<1, 2, [   // Reduction of an integer and vector into an integer
942  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>
943]>;
944def ARMVMINVu       : SDNode<"ARMISD::VMINVu", SDTVecReduceR>;
945def ARMVMINVs       : SDNode<"ARMISD::VMINVs", SDTVecReduceR>;
946def ARMVMAXVu       : SDNode<"ARMISD::VMAXVu", SDTVecReduceR>;
947def ARMVMAXVs       : SDNode<"ARMISD::VMAXVs", SDTVecReduceR>;
948
949defm MVE_VMINV : MVE_VMINMAXV_ty<"vminv", 1, "int_arm_mve_minv">;
950defm MVE_VMAXV : MVE_VMINMAXV_ty<"vmaxv", 0, "int_arm_mve_maxv">;
951
952let Predicates = [HasMVEInt] in {
953  def : Pat<(i32 (vecreduce_smax (v16i8 MQPR:$src))),
954            (i32 (MVE_VMAXVs8 (t2MVNi (i32 127)), $src))>;
955  def : Pat<(i32 (vecreduce_smax (v8i16 MQPR:$src))),
956            (i32 (MVE_VMAXVs16 (t2MOVi32imm (i32 -32768)), $src))>;
957  def : Pat<(i32 (vecreduce_smax (v4i32 MQPR:$src))),
958            (i32 (MVE_VMAXVs32 (t2MOVi (i32 -2147483648)), $src))>;
959  def : Pat<(i32 (vecreduce_umax (v16i8 MQPR:$src))),
960            (i32 (MVE_VMAXVu8 (t2MOVi (i32 0)), $src))>;
961  def : Pat<(i32 (vecreduce_umax (v8i16 MQPR:$src))),
962            (i32 (MVE_VMAXVu16 (t2MOVi (i32 0)), $src))>;
963  def : Pat<(i32 (vecreduce_umax (v4i32 MQPR:$src))),
964            (i32 (MVE_VMAXVu32 (t2MOVi (i32 0)), $src))>;
965
966  def : Pat<(i32 (vecreduce_smin (v16i8 MQPR:$src))),
967            (i32 (MVE_VMINVs8 (t2MOVi (i32 127)), $src))>;
968  def : Pat<(i32 (vecreduce_smin (v8i16 MQPR:$src))),
969            (i32 (MVE_VMINVs16 (t2MOVi16 (i32 32767)), $src))>;
970  def : Pat<(i32 (vecreduce_smin (v4i32 MQPR:$src))),
971            (i32 (MVE_VMINVs32 (t2MVNi (i32 -2147483648)), $src))>;
972  def : Pat<(i32 (vecreduce_umin (v16i8 MQPR:$src))),
973            (i32 (MVE_VMINVu8 (t2MOVi (i32 255)), $src))>;
974  def : Pat<(i32 (vecreduce_umin (v8i16 MQPR:$src))),
975            (i32 (MVE_VMINVu16 (t2MOVi16 (i32 65535)), $src))>;
976  def : Pat<(i32 (vecreduce_umin (v4i32 MQPR:$src))),
977            (i32 (MVE_VMINVu32 (t2MOVi (i32 4294967295)), $src))>;
978
979  def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v16i8 MQPR:$src))),
980            (i32 (MVE_VMINVu8 $x, $src))>;
981  def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v8i16 MQPR:$src))),
982            (i32 (MVE_VMINVu16 $x, $src))>;
983  def : Pat<(i32 (ARMVMINVu (i32 rGPR:$x), (v4i32 MQPR:$src))),
984            (i32 (MVE_VMINVu32 $x, $src))>;
985  def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v16i8 MQPR:$src))),
986            (i32 (MVE_VMINVs8 $x, $src))>;
987  def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v8i16 MQPR:$src))),
988            (i32 (MVE_VMINVs16 $x, $src))>;
989  def : Pat<(i32 (ARMVMINVs (i32 rGPR:$x), (v4i32 MQPR:$src))),
990            (i32 (MVE_VMINVs32 $x, $src))>;
991
992  def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v16i8 MQPR:$src))),
993            (i32 (MVE_VMAXVu8 $x, $src))>;
994  def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v8i16 MQPR:$src))),
995            (i32 (MVE_VMAXVu16 $x, $src))>;
996  def : Pat<(i32 (ARMVMAXVu (i32 rGPR:$x), (v4i32 MQPR:$src))),
997            (i32 (MVE_VMAXVu32 $x, $src))>;
998  def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v16i8 MQPR:$src))),
999            (i32 (MVE_VMAXVs8 $x, $src))>;
1000  def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v8i16 MQPR:$src))),
1001            (i32 (MVE_VMAXVs16 $x, $src))>;
1002  def : Pat<(i32 (ARMVMAXVs (i32 rGPR:$x), (v4i32 MQPR:$src))),
1003            (i32 (MVE_VMAXVs32 $x, $src))>;
1004
1005}
1006
1007multiclass MVE_VMINMAXAV_ty<string iname, bit isMin, string intrBaseName> {
1008  defm s8 : MVE_VMINMAXV_p<iname, 0, isMin, MVE_v16s8, intrBaseName>;
1009  defm s16: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v8s16, intrBaseName>;
1010  defm s32: MVE_VMINMAXV_p<iname, 0, isMin, MVE_v4s32, intrBaseName>;
1011}
1012
1013defm MVE_VMINAV : MVE_VMINMAXAV_ty<"vminav", 1, "int_arm_mve_minav">;
1014defm MVE_VMAXAV : MVE_VMINMAXAV_ty<"vmaxav", 0, "int_arm_mve_maxav">;
1015
1016class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,
1017                   bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
1018                   bits<2> vecsize>
1019  : MVE_rDest<(outs tGPREven:$RdaDest), iops, NoItinerary, iname, suffix,
1020              "$RdaDest, $Qn, $Qm", cstr, vecsize, []> {
1021  bits<4> RdaDest;
1022  bits<3> Qm;
1023  bits<3> Qn;
1024
1025  let Inst{28} = bit_28;
1026  let Inst{22-20} = 0b111;
1027  let Inst{19-17} = Qn{2-0};
1028  let Inst{16} = sz;
1029  let Inst{15-13} = RdaDest{3-1};
1030  let Inst{12} = X;
1031  let Inst{8} = bit_8;
1032  let Inst{7-6} = 0b00;
1033  let Inst{5} = A;
1034  let Inst{3-1} = Qm{2-0};
1035  let Inst{0} = bit_0;
1036  let horizontalReduction = 1;
1037  // Allow tail predication for non-exchanging versions. As this is also a
1038  // horizontalReduction, ARMLowOverheadLoops will also have to check that
1039  // the vector operands contain zeros in their false lanes for the instruction
1040  // to be properly valid.
1041  let validForTailPredication = !eq(X, 0);
1042}
1043
1044multiclass MVE_VMLAMLSDAV_A<string iname, string x, MVEVectorVTInfo VTI,
1045                            bit sz, bit bit_28, bit X, bit bit_8, bit bit_0> {
1046  def ""#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # x, VTI.Suffix,
1047                                   (ins MQPR:$Qn, MQPR:$Qm), "",
1048                                   sz, bit_28, 0b0, X, bit_8, bit_0, VTI.Size>;
1049  def "a"#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # "a" # x, VTI.Suffix,
1050                                    (ins tGPREven:$RdaSrc, MQPR:$Qn, MQPR:$Qm),
1051                                    "$RdaDest = $RdaSrc",
1052                                    sz, bit_28, 0b1, X, bit_8, bit_0, VTI.Size>;
1053  let Predicates = [HasMVEInt] in {
1054    def : Pat<(i32 (int_arm_mve_vmldava
1055                            (i32 VTI.Unsigned),
1056                            (i32 bit_0) /* subtract */,
1057                            (i32 X) /* exchange */,
1058                            (i32 0) /* accumulator */,
1059                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
1060              (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
1061                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
1062
1063    def : Pat<(i32 (int_arm_mve_vmldava_predicated
1064                            (i32 VTI.Unsigned),
1065                            (i32 bit_0) /* subtract */,
1066                            (i32 X) /* exchange */,
1067                            (i32 0) /* accumulator */,
1068                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1069                            (VTI.Pred VCCR:$mask))),
1070              (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
1071                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1072                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
1073
1074    def : Pat<(i32 (int_arm_mve_vmldava
1075                            (i32 VTI.Unsigned),
1076                            (i32 bit_0) /* subtract */,
1077                            (i32 X) /* exchange */,
1078                            (i32 tGPREven:$RdaSrc),
1079                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
1080              (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
1081                            (i32 tGPREven:$RdaSrc),
1082                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
1083
1084    def : Pat<(i32 (int_arm_mve_vmldava_predicated
1085                            (i32 VTI.Unsigned),
1086                            (i32 bit_0) /* subtract */,
1087                            (i32 X) /* exchange */,
1088                            (i32 tGPREven:$RdaSrc),
1089                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1090                            (VTI.Pred VCCR:$mask))),
1091              (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
1092                            (i32 tGPREven:$RdaSrc),
1093                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1094                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
1095  }
1096}
1097
1098multiclass MVE_VMLAMLSDAV_AX<string iname, MVEVectorVTInfo VTI, bit sz,
1099                             bit bit_28, bit bit_8, bit bit_0> {
1100  defm "" : MVE_VMLAMLSDAV_A<iname, "", VTI, sz, bit_28,
1101                             0b0, bit_8, bit_0>;
1102  defm "" : MVE_VMLAMLSDAV_A<iname, "x", VTI, sz, bit_28,
1103                             0b1, bit_8, bit_0>;
1104}
1105
1106multiclass MVE_VMLADAV_multi<MVEVectorVTInfo SVTI, MVEVectorVTInfo UVTI,
1107                             bit sz, bit bit_8> {
1108  defm "" : MVE_VMLAMLSDAV_AX<"vmladav", SVTI,
1109                              sz, 0b0, bit_8, 0b0>;
1110  defm "" : MVE_VMLAMLSDAV_A<"vmladav", "", UVTI,
1111                             sz, 0b1, 0b0, bit_8, 0b0>;
1112}
1113
1114multiclass MVE_VMLSDAV_multi<MVEVectorVTInfo VTI, bit sz, bit bit_28> {
1115  defm "" : MVE_VMLAMLSDAV_AX<"vmlsdav", VTI,
1116                              sz, bit_28, 0b0, 0b1>;
1117}
1118
1119defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v16s8, MVE_v16u8, 0b0, 0b1>;
1120defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v8s16, MVE_v8u16, 0b0, 0b0>;
1121defm MVE_VMLADAV : MVE_VMLADAV_multi<MVE_v4s32, MVE_v4u32, 0b1, 0b0>;
1122
1123defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v16s8, 0b0, 0b1>;
1124defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v8s16, 0b0, 0b0>;
1125defm MVE_VMLSDAV : MVE_VMLSDAV_multi<MVE_v4s32, 0b1, 0b0>;
1126
1127def SDTVecReduce2 : SDTypeProfile<1, 2, [    // VMLAV
1128  SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>
1129]>;
1130def SDTVecReduce2L : SDTypeProfile<2, 2, [    // VMLALV
1131  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>
1132]>;
1133def SDTVecReduce2LA : SDTypeProfile<2, 4, [    // VMLALVA
1134  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
1135  SDTCisVec<4>, SDTCisVec<5>
1136]>;
1137def SDTVecReduce2P : SDTypeProfile<1, 3, [    // VMLAV
1138  SDTCisInt<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>
1139]>;
1140def SDTVecReduce2LP : SDTypeProfile<2, 3, [    // VMLALV
1141  SDTCisInt<0>, SDTCisInt<1>, SDTCisVec<2>, SDTCisVec<3>, SDTCisVec<4>
1142]>;
1143def SDTVecReduce2LAP : SDTypeProfile<2, 5, [    // VMLALVA
1144  SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2>, SDTCisInt<3>,
1145  SDTCisVec<4>, SDTCisVec<5>, SDTCisVec<6>
1146]>;
1147def ARMVMLAVs       : SDNode<"ARMISD::VMLAVs", SDTVecReduce2>;
1148def ARMVMLAVu       : SDNode<"ARMISD::VMLAVu", SDTVecReduce2>;
1149def ARMVMLALVs      : SDNode<"ARMISD::VMLALVs", SDTVecReduce2L>;
1150def ARMVMLALVu      : SDNode<"ARMISD::VMLALVu", SDTVecReduce2L>;
1151def ARMVMLALVAs     : SDNode<"ARMISD::VMLALVAs", SDTVecReduce2LA>;
1152def ARMVMLALVAu     : SDNode<"ARMISD::VMLALVAu", SDTVecReduce2LA>;
1153def ARMVMLAVps      : SDNode<"ARMISD::VMLAVps", SDTVecReduce2P>;
1154def ARMVMLAVpu      : SDNode<"ARMISD::VMLAVpu", SDTVecReduce2P>;
1155def ARMVMLALVps     : SDNode<"ARMISD::VMLALVps", SDTVecReduce2LP>;
1156def ARMVMLALVpu     : SDNode<"ARMISD::VMLALVpu", SDTVecReduce2LP>;
1157def ARMVMLALVAps    : SDNode<"ARMISD::VMLALVAps", SDTVecReduce2LAP>;
1158def ARMVMLALVApu    : SDNode<"ARMISD::VMLALVApu", SDTVecReduce2LAP>;
1159
1160let Predicates = [HasMVEInt] in {
1161  def : Pat<(i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
1162            (i32 (MVE_VMLADAVu32 $src1, $src2))>;
1163  def : Pat<(i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
1164            (i32 (MVE_VMLADAVu16 $src1, $src2))>;
1165  def : Pat<(i32 (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1166            (i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1167  def : Pat<(i32 (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1168            (i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1169  def : Pat<(i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
1170            (i32 (MVE_VMLADAVu8 $src1, $src2))>;
1171  def : Pat<(i32 (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1172            (i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1173  def : Pat<(i32 (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1174            (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1175
1176  def : Pat<(i32 (add (i32 (vecreduce_add (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)))),
1177                      (i32 tGPREven:$src3))),
1178            (i32 (MVE_VMLADAVau32 $src3, $src1, $src2))>;
1179  def : Pat<(i32 (add (i32 (vecreduce_add (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)))),
1180                      (i32 tGPREven:$src3))),
1181            (i32 (MVE_VMLADAVau16 $src3, $src1, $src2))>;
1182  def : Pat<(i32 (add (ARMVMLAVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
1183            (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1184  def : Pat<(i32 (add (ARMVMLAVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)), tGPREven:$Rd)),
1185            (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1186  def : Pat<(i32 (add (i32 (vecreduce_add (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)))),
1187                      (i32 tGPREven:$src3))),
1188            (i32 (MVE_VMLADAVau8 $src3, $src1, $src2))>;
1189  def : Pat<(i32 (add (ARMVMLAVs (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
1190            (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1191  def : Pat<(i32 (add (ARMVMLAVu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)), tGPREven:$Rd)),
1192            (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1193
1194  // Predicated
1195  def : Pat<(i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),
1196                                         (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),
1197                                         (v4i32 ARMimmAllZerosV)))),
1198            (i32 (MVE_VMLADAVu32 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
1199  def : Pat<(i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),
1200                                         (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),
1201                                         (v8i16 ARMimmAllZerosV)))),
1202            (i32 (MVE_VMLADAVu16 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
1203  def : Pat<(i32 (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),
1204            (i32 (MVE_VMLADAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1205  def : Pat<(i32 (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred))),
1206            (i32 (MVE_VMLADAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1207  def : Pat<(i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),
1208                                         (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),
1209                                         (v16i8 ARMimmAllZerosV)))),
1210            (i32 (MVE_VMLADAVu8 $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
1211  def : Pat<(i32 (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),
1212            (i32 (MVE_VMLADAVs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1213  def : Pat<(i32 (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred))),
1214            (i32 (MVE_VMLADAVu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1215
1216  def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v4i1 VCCR:$pred),
1217                                                   (mul (v4i32 MQPR:$src1), (v4i32 MQPR:$src2)),
1218                                                   (v4i32 ARMimmAllZerosV)))),
1219                      (i32 tGPREven:$src3))),
1220            (i32 (MVE_VMLADAVau32 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
1221  def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v8i1 VCCR:$pred),
1222                                                   (mul (v8i16 MQPR:$src1), (v8i16 MQPR:$src2)),
1223                                                   (v8i16 ARMimmAllZerosV)))),
1224                      (i32 tGPREven:$src3))),
1225            (i32 (MVE_VMLADAVau16 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
1226  def : Pat<(i32 (add (ARMVMLAVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),
1227            (i32 (MVE_VMLADAVas16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1228  def : Pat<(i32 (add (ARMVMLAVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)), tGPREven:$Rd)),
1229            (i32 (MVE_VMLADAVau16 tGPREven:$Rd, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1230  def : Pat<(i32 (add (i32 (vecreduce_add (vselect (v16i1 VCCR:$pred),
1231                                                   (mul (v16i8 MQPR:$src1), (v16i8 MQPR:$src2)),
1232                                                   (v16i8 ARMimmAllZerosV)))),
1233                      (i32 tGPREven:$src3))),
1234            (i32 (MVE_VMLADAVau8 $src3, $src1, $src2, ARMVCCThen, $pred, zero_reg))>;
1235  def : Pat<(i32 (add (ARMVMLAVps (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),
1236            (i32 (MVE_VMLADAVas8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1237  def : Pat<(i32 (add (ARMVMLAVpu (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), (v16i1 VCCR:$pred)), tGPREven:$Rd)),
1238            (i32 (MVE_VMLADAVau8 tGPREven:$Rd, (v16i8 MQPR:$val1), (v16i8 MQPR:$val2), ARMVCCThen, $pred, zero_reg))>;
1239}
1240
1241// vmlav aliases vmladav
1242foreach acc = ["", "a"] in {
1243  foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32"] in {
1244    def : MVEInstAlias<"vmlav"#acc#"${vp}."#suffix#"\t$RdaDest, $Qn, $Qm",
1245                       (!cast<Instruction>("MVE_VMLADAV"#acc#suffix)
1246                        tGPREven:$RdaDest, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1247  }
1248}
1249
1250// Base class for VMLALDAV and VMLSLDAV, VRMLALDAVH, VRMLSLDAVH
1251class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,
1252                       bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
1253                       bits<2> vecsize, list<dag> pattern=[]>
1254  : MVE_rDest<(outs tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest), iops, NoItinerary,
1255              iname, suffix, "$RdaLoDest, $RdaHiDest, $Qn, $Qm", cstr, vecsize, pattern> {
1256  bits<4> RdaLoDest;
1257  bits<4> RdaHiDest;
1258  bits<3> Qm;
1259  bits<3> Qn;
1260
1261  let Inst{28} = bit_28;
1262  let Inst{22-20} = RdaHiDest{3-1};
1263  let Inst{19-17} = Qn{2-0};
1264  let Inst{16} = sz;
1265  let Inst{15-13} = RdaLoDest{3-1};
1266  let Inst{12} = X;
1267  let Inst{8} = bit_8;
1268  let Inst{7-6} = 0b00;
1269  let Inst{5} = A;
1270  let Inst{3-1} = Qm{2-0};
1271  let Inst{0} = bit_0;
1272  let horizontalReduction = 1;
1273  // Allow tail predication for non-exchanging versions. As this is also a
1274  // horizontalReduction, ARMLowOverheadLoops will also have to check that
1275  // the vector operands contain zeros in their false lanes for the instruction
1276  // to be properly valid.
1277  let validForTailPredication = !eq(X, 0);
1278
1279  let hasSideEffects = 0;
1280}
1281
1282multiclass MVE_VMLALDAVBase_A<string iname, string x, string suffix,
1283                              bit sz, bit bit_28, bit X, bit bit_8, bit bit_0,
1284                              bits<2> vecsize, list<dag> pattern=[]> {
1285  def ""#x#suffix : MVE_VMLALDAVBase<
1286     iname # x, suffix, (ins MQPR:$Qn, MQPR:$Qm), "",
1287     sz, bit_28, 0b0, X, bit_8, bit_0, vecsize, pattern>;
1288  def "a"#x#suffix : MVE_VMLALDAVBase<
1289     iname # "a" # x, suffix,
1290     (ins tGPREven:$RdaLoSrc, tGPROdd:$RdaHiSrc, MQPR:$Qn, MQPR:$Qm),
1291     "$RdaLoDest = $RdaLoSrc,$RdaHiDest = $RdaHiSrc",
1292     sz, bit_28, 0b1, X, bit_8, bit_0, vecsize, pattern>;
1293}
1294
1295
1296multiclass MVE_VMLALDAVBase_AX<string iname, string suffix, bit sz, bit bit_28,
1297                               bit bit_8, bit bit_0, bits<2> vecsize, list<dag> pattern=[]> {
1298  defm "" : MVE_VMLALDAVBase_A<iname, "", suffix, sz,
1299                               bit_28, 0b0, bit_8, bit_0, vecsize, pattern>;
1300  defm "" : MVE_VMLALDAVBase_A<iname, "x", suffix, sz,
1301                               bit_28, 0b1, bit_8, bit_0, vecsize, pattern>;
1302}
1303
1304multiclass MVE_VRMLALDAVH_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {
1305  defm "" : MVE_VMLALDAVBase_AX<"vrmlaldavh", "s"#VTI.BitsSuffix,
1306                                0b0, 0b0, 0b1, 0b0, VTI.Size, pattern>;
1307  defm "" : MVE_VMLALDAVBase_A<"vrmlaldavh", "", "u"#VTI.BitsSuffix,
1308                               0b0, 0b1, 0b0, 0b1, 0b0, VTI.Size, pattern>;
1309}
1310
1311defm MVE_VRMLALDAVH : MVE_VRMLALDAVH_multi<MVE_v4i32>;
1312
1313// vrmlalvh aliases for vrmlaldavh
1314def : MVEInstAlias<"vrmlalvh${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
1315                  (MVE_VRMLALDAVHs32
1316                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1317                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1318def : MVEInstAlias<"vrmlalvha${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
1319                  (MVE_VRMLALDAVHas32
1320                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1321                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1322def : MVEInstAlias<"vrmlalvh${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
1323                  (MVE_VRMLALDAVHu32
1324                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1325                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1326def : MVEInstAlias<"vrmlalvha${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
1327                  (MVE_VRMLALDAVHau32
1328                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
1329                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1330
1331multiclass MVE_VMLALDAV_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {
1332  defm "" : MVE_VMLALDAVBase_AX<"vmlaldav", "s"#VTI.BitsSuffix,
1333                                VTI.Size{1}, 0b0, 0b0, 0b0, VTI.Size, pattern>;
1334  defm "" : MVE_VMLALDAVBase_A<"vmlaldav", "", "u"#VTI.BitsSuffix,
1335                               VTI.Size{1}, 0b1, 0b0, 0b0, 0b0, VTI.Size, pattern>;
1336}
1337
1338defm MVE_VMLALDAV : MVE_VMLALDAV_multi<MVE_v8i16>;
1339defm MVE_VMLALDAV : MVE_VMLALDAV_multi<MVE_v4i32>;
1340
1341let Predicates = [HasMVEInt] in {
1342  def : Pat<(ARMVMLALVs (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1343            (MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1344  def : Pat<(ARMVMLALVu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1345            (MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1346  def : Pat<(ARMVMLALVs (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1347            (MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1348  def : Pat<(ARMVMLALVu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1349            (MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1350
1351  def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1352            (MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1353  def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)),
1354            (MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))>;
1355  def : Pat<(ARMVMLALVAs tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1356            (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1357  def : Pat<(ARMVMLALVAu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)),
1358            (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))>;
1359
1360  // Predicated
1361  def : Pat<(ARMVMLALVps (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
1362            (MVE_VMLALDAVs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1363  def : Pat<(ARMVMLALVpu (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
1364            (MVE_VMLALDAVu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1365  def : Pat<(ARMVMLALVps (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
1366            (MVE_VMLALDAVs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1367  def : Pat<(ARMVMLALVpu (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
1368            (MVE_VMLALDAVu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1369
1370  def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
1371            (MVE_VMLALDAVas32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1372  def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), (v4i1 VCCR:$pred)),
1373            (MVE_VMLALDAVau32 tGPREven:$Rda, tGPROdd:$Rdb, (v4i32 MQPR:$val1), (v4i32 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1374  def : Pat<(ARMVMLALVAps tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
1375            (MVE_VMLALDAVas16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1376  def : Pat<(ARMVMLALVApu tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), (v8i1 VCCR:$pred)),
1377            (MVE_VMLALDAVau16 tGPREven:$Rda, tGPROdd:$Rdb, (v8i16 MQPR:$val1), (v8i16 MQPR:$val2), ARMVCCThen, $pred, zero_reg)>;
1378}
1379
1380// vmlalv aliases vmlaldav
1381foreach acc = ["", "a"] in {
1382  foreach suffix = ["s16", "s32", "u16", "u32"] in {
1383    def : MVEInstAlias<"vmlalv" # acc # "${vp}." # suffix #
1384                          "\t$RdaLoDest, $RdaHiDest, $Qn, $Qm",
1385                       (!cast<Instruction>("MVE_VMLALDAV"#acc#suffix)
1386                       tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest,
1387                       MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
1388  }
1389}
1390
1391multiclass MVE_VMLSLDAV_multi<string iname, string suffix, bit sz,
1392                              bit bit_28, bits<2> vecsize, list<dag> pattern=[]> {
1393  defm "" : MVE_VMLALDAVBase_AX<iname, suffix, sz, bit_28, 0b0, 0b1, vecsize, pattern>;
1394}
1395
1396defm MVE_VMLSLDAV   : MVE_VMLSLDAV_multi<"vmlsldav", "s16", 0b0, 0b0, 0b01>;
1397defm MVE_VMLSLDAV   : MVE_VMLSLDAV_multi<"vmlsldav", "s32", 0b1, 0b0, 0b10>;
1398defm MVE_VRMLSLDAVH : MVE_VMLSLDAV_multi<"vrmlsldavh", "s32", 0b0, 0b1, 0b10>;
1399
1400// end of mve_rDest instructions
1401
1402// start of mve_comp instructions
1403
1404class MVE_comp<InstrItinClass itin, string iname, string suffix,
1405               string cstr, bits<2> vecsize, list<dag> pattern=[]>
1406  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), itin, iname, suffix,
1407           "$Qd, $Qn, $Qm", vpred_r, cstr, vecsize, pattern> {
1408  bits<4> Qd;
1409  bits<4> Qn;
1410  bits<4> Qm;
1411
1412  let Inst{22} = Qd{3};
1413  let Inst{19-17} = Qn{2-0};
1414  let Inst{16} = 0b0;
1415  let Inst{15-13} = Qd{2-0};
1416  let Inst{12} = 0b0;
1417  let Inst{10-9} = 0b11;
1418  let Inst{7} = Qn{3};
1419  let Inst{5} = Qm{3};
1420  let Inst{3-1} = Qm{2-0};
1421  let Inst{0} = 0b0;
1422}
1423
1424class MVE_VMINMAXNM<string iname, string suffix, bits<2> sz, bit bit_21,
1425                    list<dag> pattern=[]>
1426  : MVE_comp<NoItinerary, iname, suffix, "", sz, pattern> {
1427
1428  let Inst{28} = 0b1;
1429  let Inst{25-24} = 0b11;
1430  let Inst{23} = 0b0;
1431  let Inst{21} = bit_21;
1432  let Inst{20} = sz{0};
1433  let Inst{11} = 0b1;
1434  let Inst{8} = 0b1;
1435  let Inst{6} = 0b1;
1436  let Inst{4} = 0b1;
1437
1438  let Predicates = [HasMVEFloat];
1439  let validForTailPredication = 1;
1440}
1441
1442multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredInt> {
1443  def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size, bit_4>;
1444
1445  let Predicates = [HasMVEFloat] in {
1446    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 0)), !cast<Instruction>(NAME)>;
1447  }
1448}
1449
1450defm MVE_VMAXNMf32 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v4f32, fmaxnum, int_arm_mve_max_predicated>;
1451defm MVE_VMAXNMf16 : MVE_VMINMAXNM_m<"vmaxnm", 0b0, MVE_v8f16, fmaxnum, int_arm_mve_max_predicated>;
1452defm MVE_VMINNMf32 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v4f32, fminnum, int_arm_mve_min_predicated>;
1453defm MVE_VMINNMf16 : MVE_VMINMAXNM_m<"vminnm", 0b1, MVE_v8f16, fminnum, int_arm_mve_min_predicated>;
1454
1455
1456class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
1457              bit bit_4, list<dag> pattern=[]>
1458  : MVE_comp<NoItinerary, iname, suffix, "", size, pattern> {
1459
1460  let Inst{28} = U;
1461  let Inst{25-24} = 0b11;
1462  let Inst{23} = 0b0;
1463  let Inst{21-20} = size{1-0};
1464  let Inst{11} = 0b0;
1465  let Inst{8} = 0b0;
1466  let Inst{6} = 0b1;
1467  let Inst{4} = bit_4;
1468  let validForTailPredication = 1;
1469}
1470
1471multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,
1472                      SDNode Op, Intrinsic PredInt> {
1473  def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;
1474
1475  let Predicates = [HasMVEInt] in {
1476    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
1477  }
1478}
1479
1480multiclass MVE_VMAX<MVEVectorVTInfo VTI>
1481  : MVE_VMINMAX_m<"vmax", 0b0, VTI, !if(VTI.Unsigned, umax, smax), int_arm_mve_max_predicated>;
1482multiclass MVE_VMIN<MVEVectorVTInfo VTI>
1483  : MVE_VMINMAX_m<"vmin", 0b1, VTI, !if(VTI.Unsigned, umin, smin), int_arm_mve_min_predicated>;
1484
1485defm MVE_VMINs8   : MVE_VMIN<MVE_v16s8>;
1486defm MVE_VMINs16  : MVE_VMIN<MVE_v8s16>;
1487defm MVE_VMINs32  : MVE_VMIN<MVE_v4s32>;
1488defm MVE_VMINu8   : MVE_VMIN<MVE_v16u8>;
1489defm MVE_VMINu16  : MVE_VMIN<MVE_v8u16>;
1490defm MVE_VMINu32  : MVE_VMIN<MVE_v4u32>;
1491
1492defm MVE_VMAXs8   : MVE_VMAX<MVE_v16s8>;
1493defm MVE_VMAXs16  : MVE_VMAX<MVE_v8s16>;
1494defm MVE_VMAXs32  : MVE_VMAX<MVE_v4s32>;
1495defm MVE_VMAXu8   : MVE_VMAX<MVE_v16u8>;
1496defm MVE_VMAXu16  : MVE_VMAX<MVE_v8u16>;
1497defm MVE_VMAXu32  : MVE_VMAX<MVE_v4u32>;
1498
1499// end of mve_comp instructions
1500
1501// start of mve_bit instructions
1502
1503class MVE_bit_arith<dag oops, dag iops, string iname, string suffix,
1504                    string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>
1505  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred_r, cstr, vecsize, pattern> {
1506  bits<4> Qd;
1507  bits<4> Qm;
1508
1509  let Inst{22} = Qd{3};
1510  let Inst{15-13} = Qd{2-0};
1511  let Inst{5} = Qm{3};
1512  let Inst{3-1} = Qm{2-0};
1513}
1514
1515def MVE_VBIC : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
1516                             "vbic", "", "$Qd, $Qn, $Qm", "", 0b00> {
1517  bits<4> Qn;
1518
1519  let Inst{28} = 0b0;
1520  let Inst{25-23} = 0b110;
1521  let Inst{21-20} = 0b01;
1522  let Inst{19-17} = Qn{2-0};
1523  let Inst{16} = 0b0;
1524  let Inst{12-8} = 0b00001;
1525  let Inst{7} = Qn{3};
1526  let Inst{6} = 0b1;
1527  let Inst{4} = 0b1;
1528  let Inst{0} = 0b0;
1529  let validForTailPredication = 1;
1530}
1531
1532class MVE_VREV<string iname, string suffix, bits<2> size, bits<2> bit_8_7,
1533               bits<2> vecsize, string cstr="">
1534  : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm), iname,
1535                  suffix, "$Qd, $Qm", cstr, vecsize> {
1536
1537  let Inst{28} = 0b1;
1538  let Inst{25-23} = 0b111;
1539  let Inst{21-20} = 0b11;
1540  let Inst{19-18} = size;
1541  let Inst{17-16} = 0b00;
1542  let Inst{12-9} = 0b0000;
1543  let Inst{8-7} = bit_8_7;
1544  let Inst{6} = 0b1;
1545  let Inst{4} = 0b0;
1546  let Inst{0} = 0b0;
1547}
1548
1549def MVE_VREV64_8  : MVE_VREV<"vrev64", "8", 0b00, 0b00, 0b11, "@earlyclobber $Qd">;
1550def MVE_VREV64_16 : MVE_VREV<"vrev64", "16", 0b01, 0b00, 0b11, "@earlyclobber $Qd">;
1551def MVE_VREV64_32 : MVE_VREV<"vrev64", "32", 0b10, 0b00, 0b11, "@earlyclobber $Qd">;
1552
1553def MVE_VREV32_8  : MVE_VREV<"vrev32", "8", 0b00, 0b01, 0b10>;
1554def MVE_VREV32_16 : MVE_VREV<"vrev32", "16", 0b01, 0b01, 0b10>;
1555
1556def MVE_VREV16_8  : MVE_VREV<"vrev16", "8", 0b00, 0b10, 0b01>;
1557
1558let Predicates = [HasMVEInt] in {
1559  def : Pat<(v8i16 (bswap (v8i16 MQPR:$src))),
1560            (v8i16 (MVE_VREV16_8 (v8i16 MQPR:$src)))>;
1561  def : Pat<(v4i32 (bswap (v4i32 MQPR:$src))),
1562            (v4i32 (MVE_VREV32_8 (v4i32 MQPR:$src)))>;
1563}
1564
1565multiclass MVE_VREV_basic_patterns<int revbits, list<MVEVectorVTInfo> VTIs,
1566                                   Instruction Inst> {
1567  defvar unpred_op = !cast<SDNode>("ARMvrev" # revbits);
1568
1569  foreach VTI = VTIs in {
1570    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src))),
1571              (VTI.Vec (Inst (VTI.Vec MQPR:$src)))>;
1572    def : Pat<(VTI.Vec (int_arm_mve_vrev_predicated (VTI.Vec MQPR:$src),
1573                  revbits, (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1574              (VTI.Vec (Inst (VTI.Vec MQPR:$src), ARMVCCThen,
1575                  (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
1576  }
1577}
1578
1579let Predicates = [HasMVEInt] in {
1580  defm: MVE_VREV_basic_patterns<64, [MVE_v4i32, MVE_v4f32], MVE_VREV64_32>;
1581  defm: MVE_VREV_basic_patterns<64, [MVE_v8i16, MVE_v8f16], MVE_VREV64_16>;
1582  defm: MVE_VREV_basic_patterns<64, [MVE_v16i8           ], MVE_VREV64_8>;
1583
1584  defm: MVE_VREV_basic_patterns<32, [MVE_v8i16, MVE_v8f16], MVE_VREV32_16>;
1585  defm: MVE_VREV_basic_patterns<32, [MVE_v16i8           ], MVE_VREV32_8>;
1586
1587  defm: MVE_VREV_basic_patterns<16, [MVE_v16i8           ], MVE_VREV16_8>;
1588}
1589
1590def MVE_VMVN : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm),
1591                             "vmvn", "", "$Qd, $Qm", "", 0b00> {
1592  let Inst{28} = 0b1;
1593  let Inst{25-23} = 0b111;
1594  let Inst{21-16} = 0b110000;
1595  let Inst{12-6} = 0b0010111;
1596  let Inst{4} = 0b0;
1597  let Inst{0} = 0b0;
1598  let validForTailPredication = 1;
1599}
1600
1601let Predicates = [HasMVEInt] in {
1602  foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in {
1603    def : Pat<(VTI.Vec (vnotq    (VTI.Vec MQPR:$val1))),
1604              (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1)))>;
1605    def : Pat<(VTI.Vec (int_arm_mve_mvn_predicated (VTI.Vec MQPR:$val1),
1606                       (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1607              (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1), ARMVCCThen,
1608                       (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
1609  }
1610}
1611
1612class MVE_bit_ops<string iname, bits<2> bit_21_20, bit bit_28>
1613  : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
1614                  iname, "", "$Qd, $Qn, $Qm", "", 0b00> {
1615  bits<4> Qn;
1616
1617  let Inst{28} = bit_28;
1618  let Inst{25-23} = 0b110;
1619  let Inst{21-20} = bit_21_20;
1620  let Inst{19-17} = Qn{2-0};
1621  let Inst{16} = 0b0;
1622  let Inst{12-8} = 0b00001;
1623  let Inst{7} = Qn{3};
1624  let Inst{6} = 0b1;
1625  let Inst{4} = 0b1;
1626  let Inst{0} = 0b0;
1627  let validForTailPredication = 1;
1628}
1629
1630def MVE_VEOR : MVE_bit_ops<"veor", 0b00, 0b1>;
1631def MVE_VORN : MVE_bit_ops<"vorn", 0b11, 0b0>;
1632def MVE_VORR : MVE_bit_ops<"vorr", 0b10, 0b0>;
1633def MVE_VAND : MVE_bit_ops<"vand", 0b00, 0b0>;
1634
1635// add ignored suffixes as aliases
1636
1637foreach s=["s8", "s16", "s32", "u8", "u16", "u32", "i8", "i16", "i32", "f16", "f32"] in {
1638  def : MVEInstAlias<"vbic${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1639        (MVE_VBIC MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1640  def : MVEInstAlias<"veor${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1641        (MVE_VEOR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1642  def : MVEInstAlias<"vorn${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1643        (MVE_VORN MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1644  def : MVEInstAlias<"vorr${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1645        (MVE_VORR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1646  def : MVEInstAlias<"vand${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1647        (MVE_VAND MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1648}
1649
1650let Predicates = [HasMVEInt] in {
1651  defm : MVE_TwoOpPattern<MVE_v16i8, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
1652  defm : MVE_TwoOpPattern<MVE_v8i16, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
1653  defm : MVE_TwoOpPattern<MVE_v4i32, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
1654  defm : MVE_TwoOpPattern<MVE_v2i64, and, int_arm_mve_and_predicated, (? ), MVE_VAND, ARMimmAllOnesV>;
1655
1656  defm : MVE_TwoOpPattern<MVE_v16i8, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
1657  defm : MVE_TwoOpPattern<MVE_v8i16, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
1658  defm : MVE_TwoOpPattern<MVE_v4i32, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
1659  defm : MVE_TwoOpPattern<MVE_v2i64, or, int_arm_mve_orr_predicated, (? ), MVE_VORR, ARMimmAllZerosV>;
1660
1661  defm : MVE_TwoOpPattern<MVE_v16i8, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
1662  defm : MVE_TwoOpPattern<MVE_v8i16, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
1663  defm : MVE_TwoOpPattern<MVE_v4i32, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
1664  defm : MVE_TwoOpPattern<MVE_v2i64, xor, int_arm_mve_eor_predicated, (? ), MVE_VEOR, ARMimmAllZerosV>;
1665
1666  defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
1667                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
1668  defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
1669                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
1670  defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
1671                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
1672  defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(and node:$LHS, (vnotq node:$RHS))>,
1673                          int_arm_mve_bic_predicated, (? ), MVE_VBIC>;
1674
1675  defm : MVE_TwoOpPattern<MVE_v16i8, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
1676                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
1677  defm : MVE_TwoOpPattern<MVE_v8i16, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
1678                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
1679  defm : MVE_TwoOpPattern<MVE_v4i32, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
1680                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
1681  defm : MVE_TwoOpPattern<MVE_v2i64, BinOpFrag<(or node:$LHS, (vnotq node:$RHS))>,
1682                          int_arm_mve_orn_predicated, (? ), MVE_VORN>;
1683}
1684
1685class MVE_bit_cmode<string iname, string suffix, bit halfword, dag inOps, bits<2> vecsize>
1686  : MVE_p<(outs MQPR:$Qd), inOps, NoItinerary,
1687          iname, suffix, "$Qd, $imm", vpred_n, "$Qd = $Qd_src", vecsize> {
1688  bits<12> imm;
1689  bits<4> Qd;
1690
1691  let Inst{28} = imm{7};
1692  let Inst{27-23} = 0b11111;
1693  let Inst{22} = Qd{3};
1694  let Inst{21-19} = 0b000;
1695  let Inst{18-16} = imm{6-4};
1696  let Inst{15-13} = Qd{2-0};
1697  let Inst{12} = 0b0;
1698  let Inst{11} = halfword;
1699  let Inst{10} = !if(halfword, 0, imm{10});
1700  let Inst{9} = imm{9};
1701  let Inst{8} = 0b1;
1702  let Inst{7-6} = 0b01;
1703  let Inst{4} = 0b1;
1704  let Inst{3-0} = imm{3-0};
1705}
1706
1707multiclass MVE_bit_cmode_p<string iname, bit opcode,
1708                           MVEVectorVTInfo VTI, Operand imm_type, SDNode op> {
1709  def "" : MVE_bit_cmode<iname, VTI.Suffix, VTI.Size{0},
1710                         (ins MQPR:$Qd_src, imm_type:$imm), VTI.Size> {
1711    let Inst{5} = opcode;
1712    let validForTailPredication = 1;
1713  }
1714
1715  defvar Inst = !cast<Instruction>(NAME);
1716  defvar UnpredPat = (VTI.Vec (op (VTI.Vec MQPR:$src), timm:$simm));
1717
1718  let Predicates = [HasMVEInt] in {
1719    def : Pat<UnpredPat,
1720              (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm))>;
1721    def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
1722                          UnpredPat, (VTI.Vec MQPR:$src))),
1723              (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm,
1724                             ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
1725  }
1726}
1727
1728multiclass MVE_VORRimm<MVEVectorVTInfo VTI, Operand imm_type> {
1729  defm "": MVE_bit_cmode_p<"vorr", 0, VTI, imm_type, ARMvorrImm>;
1730}
1731multiclass MVE_VBICimm<MVEVectorVTInfo VTI, Operand imm_type> {
1732  defm "": MVE_bit_cmode_p<"vbic", 1, VTI, imm_type, ARMvbicImm>;
1733}
1734
1735defm MVE_VORRimmi16 : MVE_VORRimm<MVE_v8i16, nImmSplatI16>;
1736defm MVE_VORRimmi32 : MVE_VORRimm<MVE_v4i32, nImmSplatI32>;
1737defm MVE_VBICimmi16 : MVE_VBICimm<MVE_v8i16, nImmSplatI16>;
1738defm MVE_VBICimmi32 : MVE_VBICimm<MVE_v4i32, nImmSplatI32>;
1739
1740def MVE_VORNimmi16 : MVEInstAlias<"vorn${vp}.i16\t$Qd, $imm",
1741    (MVE_VORRimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;
1742def MVE_VORNimmi32 : MVEInstAlias<"vorn${vp}.i32\t$Qd, $imm",
1743    (MVE_VORRimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;
1744
1745def MVE_VANDimmi16 : MVEInstAlias<"vand${vp}.i16\t$Qd, $imm",
1746    (MVE_VBICimmi16 MQPR:$Qd, nImmSplatNotI16:$imm, vpred_n:$vp), 0>;
1747def MVE_VANDimmi32 : MVEInstAlias<"vand${vp}.i32\t$Qd, $imm",
1748    (MVE_VBICimmi32 MQPR:$Qd, nImmSplatNotI32:$imm, vpred_n:$vp), 0>;
1749
1750def MVE_VMOV : MVEInstAlias<"vmov${vp}\t$Qd, $Qm",
1751    (MVE_VORR MQPR:$Qd, MQPR:$Qm, MQPR:$Qm, vpred_r:$vp)>;
1752
1753class MVE_VMOV_lane_direction {
1754  bit bit_20;
1755  dag oops;
1756  dag iops;
1757  string ops;
1758  string cstr;
1759}
1760def MVE_VMOV_from_lane : MVE_VMOV_lane_direction {
1761  let bit_20 = 0b1;
1762  let oops = (outs rGPR:$Rt);
1763  let iops = (ins MQPR:$Qd);
1764  let ops = "$Rt, $Qd$Idx";
1765  let cstr = "";
1766}
1767def MVE_VMOV_to_lane : MVE_VMOV_lane_direction {
1768  let bit_20 = 0b0;
1769  let oops = (outs MQPR:$Qd);
1770  let iops = (ins MQPR:$Qd_src, rGPR:$Rt);
1771  let ops = "$Qd$Idx, $Rt";
1772  let cstr = "$Qd = $Qd_src";
1773}
1774
1775class MVE_VMOV_lane<string suffix, bit U, dag indexop,
1776                    MVE_VMOV_lane_direction dir>
1777  : MVE_VMOV_lane_base<dir.oops, !con(dir.iops, indexop), NoItinerary,
1778                       "vmov", suffix, dir.ops, dir.cstr, []> {
1779  bits<4> Qd;
1780  bits<4> Rt;
1781
1782  let Inst{31-24} = 0b11101110;
1783  let Inst{23} = U;
1784  let Inst{20} = dir.bit_20;
1785  let Inst{19-17} = Qd{2-0};
1786  let Inst{15-12} = Rt{3-0};
1787  let Inst{11-8} = 0b1011;
1788  let Inst{7} = Qd{3};
1789  let Inst{4-0} = 0b10000;
1790
1791  let hasSideEffects = 0;
1792}
1793
1794class MVE_VMOV_lane_32<MVE_VMOV_lane_direction dir>
1795    : MVE_VMOV_lane<"32", 0b0, (ins MVEVectorIndex<4>:$Idx), dir> {
1796  bits<2> Idx;
1797  let Inst{22} = 0b0;
1798  let Inst{6-5} = 0b00;
1799  let Inst{16} = Idx{1};
1800  let Inst{21} = Idx{0};
1801
1802  let VecSize = 0b10;
1803  let Predicates = [HasFPRegsV8_1M];
1804}
1805
1806class MVE_VMOV_lane_16<string suffix, bit U, MVE_VMOV_lane_direction dir>
1807  : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<8>:$Idx), dir> {
1808  bits<3> Idx;
1809  let Inst{22} = 0b0;
1810  let Inst{5} = 0b1;
1811  let Inst{16} = Idx{2};
1812  let Inst{21} = Idx{1};
1813  let Inst{6} = Idx{0};
1814
1815  let VecSize = 0b01;
1816}
1817
1818class MVE_VMOV_lane_8<string suffix, bit U, MVE_VMOV_lane_direction dir>
1819  : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<16>:$Idx), dir> {
1820  bits<4> Idx;
1821  let Inst{22} = 0b1;
1822  let Inst{16} = Idx{3};
1823  let Inst{21} = Idx{2};
1824  let Inst{6} = Idx{1};
1825  let Inst{5} = Idx{0};
1826
1827  let VecSize = 0b00;
1828}
1829
1830def MVE_VMOV_from_lane_32  : MVE_VMOV_lane_32<            MVE_VMOV_from_lane>;
1831def MVE_VMOV_from_lane_s16 : MVE_VMOV_lane_16<"s16", 0b0, MVE_VMOV_from_lane>;
1832def MVE_VMOV_from_lane_u16 : MVE_VMOV_lane_16<"u16", 0b1, MVE_VMOV_from_lane>;
1833def MVE_VMOV_from_lane_s8  : MVE_VMOV_lane_8 < "s8", 0b0, MVE_VMOV_from_lane>;
1834def MVE_VMOV_from_lane_u8  : MVE_VMOV_lane_8 < "u8", 0b1, MVE_VMOV_from_lane>;
1835let isInsertSubreg = 1 in
1836def MVE_VMOV_to_lane_32    : MVE_VMOV_lane_32<            MVE_VMOV_to_lane>;
1837def MVE_VMOV_to_lane_16    : MVE_VMOV_lane_16< "16", 0b0, MVE_VMOV_to_lane>;
1838def MVE_VMOV_to_lane_8     : MVE_VMOV_lane_8 <  "8", 0b0, MVE_VMOV_to_lane>;
1839
1840// This is the same as insertelt but allows the inserted value to be an i32 as
1841// will be used when it is the only legal type.
1842def ARMVecInsert : SDTypeProfile<1, 3, [
1843  SDTCisVT<2, i32>, SDTCisSameAs<0, 1>, SDTCisPtrTy<3>
1844]>;
1845def ARMinsertelt  : SDNode<"ISD::INSERT_VECTOR_ELT", ARMVecInsert>;
1846
1847let Predicates = [HasMVEInt] in {
1848  def : Pat<(extractelt (v2f64 MQPR:$src), imm:$lane),
1849            (f64 (EXTRACT_SUBREG MQPR:$src, (DSubReg_f64_reg imm:$lane)))>;
1850  def : Pat<(insertelt (v2f64 MQPR:$src1), DPR:$src2, imm:$lane),
1851            (INSERT_SUBREG (v2f64 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), DPR:$src2, (DSubReg_f64_reg imm:$lane))>;
1852
1853  def : Pat<(extractelt (v4i32 MQPR:$src), imm:$lane),
1854            (COPY_TO_REGCLASS
1855              (i32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), rGPR)>;
1856  def : Pat<(insertelt (v4i32 MQPR:$src1), rGPR:$src2, imm:$lane),
1857            (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$src2, imm:$lane)>;
1858  // This tries to copy from one lane to another, without going via GPR regs
1859  def : Pat<(insertelt (v4i32 MQPR:$src1), (extractelt (v4i32 MQPR:$src2), imm:$extlane), imm:$inslane),
1860            (v4i32 (COPY_TO_REGCLASS
1861                     (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS (v4i32 MQPR:$src1), MQPR)),
1862                                    (f32 (EXTRACT_SUBREG (v4f32 (COPY_TO_REGCLASS (v4i32 MQPR:$src2), MQPR)),
1863                                                         (SSubReg_f32_reg imm:$extlane))),
1864                                    (SSubReg_f32_reg imm:$inslane)),
1865                      MQPR))>;
1866
1867  def : Pat<(vector_insert (v16i8 MQPR:$src1), rGPR:$src2, imm:$lane),
1868            (MVE_VMOV_to_lane_8  MQPR:$src1, rGPR:$src2, imm:$lane)>;
1869  def : Pat<(vector_insert (v8i16 MQPR:$src1), rGPR:$src2, imm:$lane),
1870            (MVE_VMOV_to_lane_16 MQPR:$src1, rGPR:$src2, imm:$lane)>;
1871
1872  def : Pat<(ARMvgetlanes (v16i8 MQPR:$src), imm:$lane),
1873            (MVE_VMOV_from_lane_s8 MQPR:$src, imm:$lane)>;
1874  def : Pat<(ARMvgetlanes (v8i16 MQPR:$src), imm:$lane),
1875            (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
1876  def : Pat<(ARMvgetlanes (v8f16 MQPR:$src), imm:$lane),
1877            (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
1878  def : Pat<(ARMvgetlaneu (v16i8 MQPR:$src), imm:$lane),
1879            (MVE_VMOV_from_lane_u8 MQPR:$src, imm:$lane)>;
1880  def : Pat<(ARMvgetlaneu (v8i16 MQPR:$src), imm:$lane),
1881            (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
1882  def : Pat<(ARMvgetlaneu (v8f16 MQPR:$src), imm:$lane),
1883            (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
1884  // For i16's inserts being extracted from low lanes, then may use VINS.
1885  let Predicates = [HasFullFP16] in {
1886  def : Pat<(ARMinsertelt (v8i16 MQPR:$src1),
1887                          (ARMvgetlaneu (v8i16 MQPR:$src2), imm_even:$extlane),
1888                          imm_odd:$inslane),
1889            (COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),
1890                                (VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$inslane)),
1891                                       (EXTRACT_SUBREG MQPR:$src2, (SSubReg_f16_reg imm_even:$extlane))),
1892                                (SSubReg_f16_reg imm_odd:$inslane)), MQPR)>;
1893  }
1894
1895  def : Pat<(v16i8 (scalar_to_vector GPR:$src)),
1896            (MVE_VMOV_to_lane_8  (v16i8 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1897  def : Pat<(v8i16 (scalar_to_vector GPR:$src)),
1898            (MVE_VMOV_to_lane_16 (v8i16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1899  def : Pat<(v4i32 (scalar_to_vector GPR:$src)),
1900            (MVE_VMOV_to_lane_32 (v4i32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1901
1902  // Floating point patterns, still enabled under HasMVEInt
1903  def : Pat<(extractelt (v4f32 MQPR:$src), imm:$lane),
1904            (COPY_TO_REGCLASS (f32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), SPR)>;
1905  def : Pat<(insertelt (v4f32 MQPR:$src1), (f32 SPR:$src2), imm:$lane),
1906            (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), SPR:$src2, (SSubReg_f32_reg imm:$lane))>;
1907
1908  def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_even:$lane),
1909            (MVE_VMOV_to_lane_16 MQPR:$src1, (COPY_TO_REGCLASS (f16 HPR:$src2), rGPR), imm:$lane)>;
1910  let Predicates = [HasFullFP16] in {
1911  def : Pat<(insertelt (v8f16 MQPR:$src1), (f16 HPR:$src2), imm_odd:$lane),
1912            (COPY_TO_REGCLASS (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)),
1913                                (VINSH (EXTRACT_SUBREG MQPR:$src1, (SSubReg_f16_reg imm_odd:$lane)),
1914                                       (COPY_TO_REGCLASS HPR:$src2, SPR)),
1915                                (SSubReg_f16_reg imm_odd:$lane)), MQPR)>;
1916  }
1917  def : Pat<(extractelt (v8f16 MQPR:$src), imm_even:$lane),
1918            (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_even:$lane))>;
1919  let Predicates = [HasFullFP16] in {
1920  def : Pat<(extractelt (v8f16 MQPR:$src), imm_odd:$lane),
1921            (COPY_TO_REGCLASS
1922              (VMOVH (EXTRACT_SUBREG MQPR:$src, (SSubReg_f16_reg imm_odd:$lane))),
1923              HPR)>;
1924  }
1925
1926  def : Pat<(v2f64 (scalar_to_vector (f64 DPR:$src))),
1927            (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), DPR:$src, dsub_0)>;
1928  def : Pat<(v4f32 (scalar_to_vector SPR:$src)),
1929            (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), SPR:$src, ssub_0)>;
1930  def : Pat<(v4f32 (scalar_to_vector GPR:$src)),
1931            (MVE_VMOV_to_lane_32 (v4f32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1932  def : Pat<(v8f16 (scalar_to_vector (f16 HPR:$src))),
1933            (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), (f16 HPR:$src), ssub_0)>;
1934  def : Pat<(v8f16 (scalar_to_vector GPR:$src)),
1935            (MVE_VMOV_to_lane_16 (v8f16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1936}
1937
1938// end of mve_bit instructions
1939
1940// start of MVE Integer instructions
1941
1942class MVE_int<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
1943  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
1944          iname, suffix, "$Qd, $Qn, $Qm", vpred_r, "", size, pattern> {
1945  bits<4> Qd;
1946  bits<4> Qn;
1947  bits<4> Qm;
1948
1949  let Inst{22} = Qd{3};
1950  let Inst{21-20} = size;
1951  let Inst{19-17} = Qn{2-0};
1952  let Inst{15-13} = Qd{2-0};
1953  let Inst{7} = Qn{3};
1954  let Inst{6} = 0b1;
1955  let Inst{5} = Qm{3};
1956  let Inst{3-1} = Qm{2-0};
1957}
1958
1959class MVE_VMULt1<string iname, string suffix, bits<2> size,
1960                   list<dag> pattern=[]>
1961  : MVE_int<iname, suffix, size, pattern> {
1962
1963  let Inst{28} = 0b0;
1964  let Inst{25-23} = 0b110;
1965  let Inst{16} = 0b0;
1966  let Inst{12-8} = 0b01001;
1967  let Inst{4} = 0b1;
1968  let Inst{0} = 0b0;
1969  let validForTailPredication = 1;
1970}
1971
1972multiclass MVE_VMUL_m<MVEVectorVTInfo VTI> {
1973  def "" : MVE_VMULt1<"vmul", VTI.Suffix, VTI.Size>;
1974
1975  let Predicates = [HasMVEInt] in {
1976    defm : MVE_TwoOpPattern<VTI, mul, int_arm_mve_mul_predicated, (? ),
1977                            !cast<Instruction>(NAME), ARMimmOneV>;
1978  }
1979}
1980
1981defm MVE_VMULi8  : MVE_VMUL_m<MVE_v16i8>;
1982defm MVE_VMULi16 : MVE_VMUL_m<MVE_v8i16>;
1983defm MVE_VMULi32 : MVE_VMUL_m<MVE_v4i32>;
1984
1985class MVE_VQxDMULH_Base<string iname, string suffix, bits<2> size, bit rounding,
1986                  list<dag> pattern=[]>
1987  : MVE_int<iname, suffix, size, pattern> {
1988
1989  let Inst{28} = rounding;
1990  let Inst{25-23} = 0b110;
1991  let Inst{16} = 0b0;
1992  let Inst{12-8} = 0b01011;
1993  let Inst{4} = 0b0;
1994  let Inst{0} = 0b0;
1995  let validForTailPredication = 1;
1996}
1997
1998def MVEvqdmulh : SDNode<"ARMISD::VQDMULH", SDTIntBinOp>;
1999
2000multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,
2001                      SDNode Op, Intrinsic unpred_int, Intrinsic pred_int,
2002                      bit rounding> {
2003  def "" : MVE_VQxDMULH_Base<iname, VTI.Suffix, VTI.Size, rounding>;
2004  defvar Inst = !cast<Instruction>(NAME);
2005
2006  let Predicates = [HasMVEInt] in {
2007    defm : MVE_TwoOpPattern<VTI, Op, pred_int, (? ), Inst>;
2008
2009    // Extra unpredicated multiply intrinsic patterns
2010    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
2011              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2012  }
2013}
2014
2015multiclass MVE_VQxDMULH<string iname, MVEVectorVTInfo VTI, bit rounding>
2016  : MVE_VQxDMULH_m<iname, VTI, !if(rounding, null_frag,
2017                                             MVEvqdmulh),
2018                               !if(rounding, int_arm_mve_vqrdmulh,
2019                                             int_arm_mve_vqdmulh),
2020                               !if(rounding, int_arm_mve_qrdmulh_predicated,
2021                                             int_arm_mve_qdmulh_predicated),
2022                   rounding>;
2023
2024defm MVE_VQDMULHi8  : MVE_VQxDMULH<"vqdmulh", MVE_v16s8, 0b0>;
2025defm MVE_VQDMULHi16 : MVE_VQxDMULH<"vqdmulh", MVE_v8s16, 0b0>;
2026defm MVE_VQDMULHi32 : MVE_VQxDMULH<"vqdmulh", MVE_v4s32, 0b0>;
2027
2028defm MVE_VQRDMULHi8  : MVE_VQxDMULH<"vqrdmulh", MVE_v16s8, 0b1>;
2029defm MVE_VQRDMULHi16 : MVE_VQxDMULH<"vqrdmulh", MVE_v8s16, 0b1>;
2030defm MVE_VQRDMULHi32 : MVE_VQxDMULH<"vqrdmulh", MVE_v4s32, 0b1>;
2031
2032class MVE_VADDSUB<string iname, string suffix, bits<2> size, bit subtract,
2033                    list<dag> pattern=[]>
2034  : MVE_int<iname, suffix, size, pattern> {
2035
2036  let Inst{28} = subtract;
2037  let Inst{25-23} = 0b110;
2038  let Inst{16} = 0b0;
2039  let Inst{12-8} = 0b01000;
2040  let Inst{4} = 0b0;
2041  let Inst{0} = 0b0;
2042  let validForTailPredication = 1;
2043}
2044
2045multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,
2046                         SDNode Op, Intrinsic PredInt> {
2047  def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;
2048  defvar Inst = !cast<Instruction>(NAME);
2049
2050  let Predicates = [HasMVEInt] in {
2051    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
2052  }
2053}
2054
2055multiclass MVE_VADD<MVEVectorVTInfo VTI>
2056  : MVE_VADDSUB_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
2057multiclass MVE_VSUB<MVEVectorVTInfo VTI>
2058  : MVE_VADDSUB_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
2059
2060defm MVE_VADDi8  : MVE_VADD<MVE_v16i8>;
2061defm MVE_VADDi16 : MVE_VADD<MVE_v8i16>;
2062defm MVE_VADDi32 : MVE_VADD<MVE_v4i32>;
2063
2064defm MVE_VSUBi8  : MVE_VSUB<MVE_v16i8>;
2065defm MVE_VSUBi16 : MVE_VSUB<MVE_v8i16>;
2066defm MVE_VSUBi32 : MVE_VSUB<MVE_v4i32>;
2067
2068class MVE_VQADDSUB<string iname, string suffix, bit U, bit subtract,
2069                   bits<2> size>
2070  : MVE_int<iname, suffix, size, []> {
2071
2072  let Inst{28} = U;
2073  let Inst{25-23} = 0b110;
2074  let Inst{16} = 0b0;
2075  let Inst{12-10} = 0b000;
2076  let Inst{9} = subtract;
2077  let Inst{8} = 0b0;
2078  let Inst{4} = 0b1;
2079  let Inst{0} = 0b0;
2080  let validForTailPredication = 1;
2081}
2082
2083class MVE_VQADD_<string suffix, bit U, bits<2> size>
2084  : MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size>;
2085class MVE_VQSUB_<string suffix, bit U, bits<2> size>
2086  : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size>;
2087
2088multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,
2089                      SDNode Op, Intrinsic PredInt> {
2090  def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2091  defvar Inst = !cast<Instruction>(NAME);
2092
2093  let Predicates = [HasMVEInt] in {
2094    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2095                            !cast<Instruction>(NAME)>;
2096  }
2097}
2098
2099multiclass MVE_VQADD<MVEVectorVTInfo VTI, SDNode unpred_op>
2100  : MVE_VQADD_m<VTI, unpred_op, int_arm_mve_qadd_predicated>;
2101
2102defm MVE_VQADDs8  : MVE_VQADD<MVE_v16s8, saddsat>;
2103defm MVE_VQADDs16 : MVE_VQADD<MVE_v8s16, saddsat>;
2104defm MVE_VQADDs32 : MVE_VQADD<MVE_v4s32, saddsat>;
2105defm MVE_VQADDu8  : MVE_VQADD<MVE_v16u8, uaddsat>;
2106defm MVE_VQADDu16 : MVE_VQADD<MVE_v8u16, uaddsat>;
2107defm MVE_VQADDu32 : MVE_VQADD<MVE_v4u32, uaddsat>;
2108
2109multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,
2110                      SDNode Op, Intrinsic PredInt> {
2111  def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2112  defvar Inst = !cast<Instruction>(NAME);
2113
2114  let Predicates = [HasMVEInt] in {
2115    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2116                            !cast<Instruction>(NAME)>;
2117  }
2118}
2119
2120multiclass MVE_VQSUB<MVEVectorVTInfo VTI, SDNode unpred_op>
2121  : MVE_VQSUB_m<VTI, unpred_op, int_arm_mve_qsub_predicated>;
2122
2123defm MVE_VQSUBs8  : MVE_VQSUB<MVE_v16s8, ssubsat>;
2124defm MVE_VQSUBs16 : MVE_VQSUB<MVE_v8s16, ssubsat>;
2125defm MVE_VQSUBs32 : MVE_VQSUB<MVE_v4s32, ssubsat>;
2126defm MVE_VQSUBu8  : MVE_VQSUB<MVE_v16u8, usubsat>;
2127defm MVE_VQSUBu16 : MVE_VQSUB<MVE_v8u16, usubsat>;
2128defm MVE_VQSUBu32 : MVE_VQSUB<MVE_v4u32, usubsat>;
2129
2130class MVE_VABD_int<string suffix, bit U, bits<2> size,
2131                     list<dag> pattern=[]>
2132  : MVE_int<"vabd", suffix, size, pattern> {
2133
2134  let Inst{28} = U;
2135  let Inst{25-23} = 0b110;
2136  let Inst{16} = 0b0;
2137  let Inst{12-8} = 0b00111;
2138  let Inst{4} = 0b0;
2139  let Inst{0} = 0b0;
2140  let validForTailPredication = 1;
2141}
2142
2143multiclass MVE_VABD_m<MVEVectorVTInfo VTI, SDNode Op,
2144                      Intrinsic unpred_int, Intrinsic PredInt> {
2145  def "" : MVE_VABD_int<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2146  defvar Inst = !cast<Instruction>(NAME);
2147
2148  let Predicates = [HasMVEInt] in {
2149    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2150                            !cast<Instruction>(NAME)>;
2151
2152    // Unpredicated absolute difference
2153    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2154                            (i32 VTI.Unsigned))),
2155              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2156  }
2157}
2158
2159multiclass MVE_VABD<MVEVectorVTInfo VTI, SDNode Op>
2160  : MVE_VABD_m<VTI, Op, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
2161
2162defm MVE_VABDs8  : MVE_VABD<MVE_v16s8, abds>;
2163defm MVE_VABDs16 : MVE_VABD<MVE_v8s16, abds>;
2164defm MVE_VABDs32 : MVE_VABD<MVE_v4s32, abds>;
2165defm MVE_VABDu8  : MVE_VABD<MVE_v16u8, abdu>;
2166defm MVE_VABDu16 : MVE_VABD<MVE_v8u16, abdu>;
2167defm MVE_VABDu32 : MVE_VABD<MVE_v4u32, abdu>;
2168
2169class MVE_VRHADD_Base<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
2170  : MVE_int<"vrhadd", suffix, size, pattern> {
2171
2172  let Inst{28} = U;
2173  let Inst{25-23} = 0b110;
2174  let Inst{16} = 0b0;
2175  let Inst{12-8} = 0b00001;
2176  let Inst{4} = 0b0;
2177  let Inst{0} = 0b0;
2178  let validForTailPredication = 1;
2179}
2180
2181def addnuw : PatFrag<(ops node:$lhs, node:$rhs),
2182                     (add node:$lhs, node:$rhs), [{
2183  return N->getFlags().hasNoUnsignedWrap();
2184}]>;
2185
2186def addnsw : PatFrag<(ops node:$lhs, node:$rhs),
2187                     (add node:$lhs, node:$rhs), [{
2188  return N->getFlags().hasNoSignedWrap();
2189}]>;
2190
2191def subnuw : PatFrag<(ops node:$lhs, node:$rhs),
2192                     (sub node:$lhs, node:$rhs), [{
2193  return N->getFlags().hasNoUnsignedWrap();
2194}]>;
2195
2196def subnsw : PatFrag<(ops node:$lhs, node:$rhs),
2197                     (sub node:$lhs, node:$rhs), [{
2198  return N->getFlags().hasNoSignedWrap();
2199}]>;
2200
2201multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI, SDNode Op,
2202                      SDNode unpred_op, Intrinsic PredInt> {
2203  def "" : MVE_VRHADD_Base<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2204  defvar Inst = !cast<Instruction>(NAME);
2205  defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
2206
2207  let Predicates = [HasMVEInt] in {
2208    // Unpredicated rounding add-with-divide-by-two intrinsic
2209    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2210                            (i32 VTI.Unsigned))),
2211              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2212  }
2213}
2214
2215multiclass MVE_VRHADD<MVEVectorVTInfo VTI, SDNode rhadd>
2216  : MVE_VRHADD_m<VTI, rhadd, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;
2217
2218defm MVE_VRHADDs8  : MVE_VRHADD<MVE_v16s8, avgceils>;
2219defm MVE_VRHADDs16 : MVE_VRHADD<MVE_v8s16, avgceils>;
2220defm MVE_VRHADDs32 : MVE_VRHADD<MVE_v4s32, avgceils>;
2221defm MVE_VRHADDu8  : MVE_VRHADD<MVE_v16u8, avgceilu>;
2222defm MVE_VRHADDu16 : MVE_VRHADD<MVE_v8u16, avgceilu>;
2223defm MVE_VRHADDu32 : MVE_VRHADD<MVE_v4u32, avgceilu>;
2224
2225// Rounding Halving Add perform the arithemtic operation with an extra bit of
2226// precision, before performing the shift, to void clipping errors. We're not
2227// modelling that here with these patterns, but we're using no wrap forms of
2228// add to ensure that the extra bit of information is not needed for the
2229// arithmetic or the rounding.
2230let Predicates = [HasMVEInt] in {
2231  def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
2232                                        (v16i8 (ARMvmovImm (i32 3585)))),
2233                                (i32 1))),
2234            (MVE_VRHADDs8 MQPR:$Qm, MQPR:$Qn)>;
2235  def : Pat<(v8i16 (ARMvshrsImm (addnsw (addnsw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
2236                                        (v8i16 (ARMvmovImm (i32 2049)))),
2237                                (i32 1))),
2238            (MVE_VRHADDs16 MQPR:$Qm, MQPR:$Qn)>;
2239  def : Pat<(v4i32 (ARMvshrsImm (addnsw (addnsw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
2240                                        (v4i32 (ARMvmovImm (i32 1)))),
2241                                (i32 1))),
2242            (MVE_VRHADDs32 MQPR:$Qm, MQPR:$Qn)>;
2243  def : Pat<(v16i8 (ARMvshruImm (addnuw (addnuw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
2244                                        (v16i8 (ARMvmovImm (i32 3585)))),
2245                                (i32 1))),
2246            (MVE_VRHADDu8 MQPR:$Qm, MQPR:$Qn)>;
2247  def : Pat<(v8i16 (ARMvshruImm (addnuw (addnuw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
2248                                        (v8i16 (ARMvmovImm (i32 2049)))),
2249                                (i32 1))),
2250            (MVE_VRHADDu16 MQPR:$Qm, MQPR:$Qn)>;
2251  def : Pat<(v4i32 (ARMvshruImm (addnuw (addnuw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
2252                                        (v4i32 (ARMvmovImm (i32 1)))),
2253                                (i32 1))),
2254            (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>;
2255
2256  def : Pat<(v16i8 (ARMvshrsImm (addnsw (addnsw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
2257                                        (v16i8 (ARMvdup (i32 1)))),
2258                                (i32 1))),
2259            (MVE_VRHADDs8 MQPR:$Qm, MQPR:$Qn)>;
2260  def : Pat<(v8i16 (ARMvshrsImm (addnsw (addnsw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
2261                                        (v8i16 (ARMvdup (i32 1)))),
2262                                (i32 1))),
2263            (MVE_VRHADDs16 MQPR:$Qm, MQPR:$Qn)>;
2264  def : Pat<(v4i32 (ARMvshrsImm (addnsw (addnsw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
2265                                        (v4i32 (ARMvdup (i32 1)))),
2266                                (i32 1))),
2267            (MVE_VRHADDs32 MQPR:$Qm, MQPR:$Qn)>;
2268  def : Pat<(v16i8 (ARMvshruImm (addnuw (addnuw (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)),
2269                                        (v16i8 (ARMvdup (i32 1)))),
2270                                (i32 1))),
2271            (MVE_VRHADDu8 MQPR:$Qm, MQPR:$Qn)>;
2272  def : Pat<(v8i16 (ARMvshruImm (addnuw (addnuw (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)),
2273                                        (v8i16 (ARMvdup (i32 1)))),
2274                                (i32 1))),
2275            (MVE_VRHADDu16 MQPR:$Qm, MQPR:$Qn)>;
2276  def : Pat<(v4i32 (ARMvshruImm (addnuw (addnuw (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)),
2277                                        (v4i32 (ARMvdup (i32 1)))),
2278                                (i32 1))),
2279            (MVE_VRHADDu32 MQPR:$Qm, MQPR:$Qn)>;
2280}
2281
2282
2283class MVE_VHADDSUB<string iname, string suffix, bit U, bit subtract,
2284                   bits<2> size, list<dag> pattern=[]>
2285  : MVE_int<iname, suffix, size, pattern> {
2286
2287  let Inst{28} = U;
2288  let Inst{25-23} = 0b110;
2289  let Inst{16} = 0b0;
2290  let Inst{12-10} = 0b000;
2291  let Inst{9} = subtract;
2292  let Inst{8} = 0b0;
2293  let Inst{4} = 0b0;
2294  let Inst{0} = 0b0;
2295  let validForTailPredication = 1;
2296}
2297
2298class MVE_VHADD_<string suffix, bit U, bits<2> size,
2299              list<dag> pattern=[]>
2300  : MVE_VHADDSUB<"vhadd", suffix, U, 0b0, size, pattern>;
2301class MVE_VHSUB_<string suffix, bit U, bits<2> size,
2302              list<dag> pattern=[]>
2303  : MVE_VHADDSUB<"vhsub", suffix, U, 0b1, size, pattern>;
2304
2305multiclass MVE_VHADD_m<MVEVectorVTInfo VTI, SDNode Op,
2306                      SDNode unpred_op, Intrinsic PredInt, PatFrag add_op,
2307                      SDNode shift_op> {
2308  def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2309  defvar Inst = !cast<Instruction>(NAME);
2310  defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
2311
2312  let Predicates = [HasMVEInt] in {
2313    // Unpredicated add-and-divide-by-two
2314    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),
2315              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2316
2317    def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2318              (Inst MQPR:$Qm, MQPR:$Qn)>;
2319  }
2320}
2321
2322multiclass MVE_VHADD<MVEVectorVTInfo VTI, SDNode Op, PatFrag add_op, SDNode shift_op>
2323  : MVE_VHADD_m<VTI, Op, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, add_op,
2324                shift_op>;
2325
2326// Halving add/sub perform the arithemtic operation with an extra bit of
2327// precision, before performing the shift, to void clipping errors. We're not
2328// modelling that here with these patterns, but we're using no wrap forms of
2329// add/sub to ensure that the extra bit of information is not needed.
2330defm MVE_VHADDs8  : MVE_VHADD<MVE_v16s8, avgfloors, addnsw, ARMvshrsImm>;
2331defm MVE_VHADDs16 : MVE_VHADD<MVE_v8s16, avgfloors, addnsw, ARMvshrsImm>;
2332defm MVE_VHADDs32 : MVE_VHADD<MVE_v4s32, avgfloors, addnsw, ARMvshrsImm>;
2333defm MVE_VHADDu8  : MVE_VHADD<MVE_v16u8, avgflooru, addnuw, ARMvshruImm>;
2334defm MVE_VHADDu16 : MVE_VHADD<MVE_v8u16, avgflooru, addnuw, ARMvshruImm>;
2335defm MVE_VHADDu32 : MVE_VHADD<MVE_v4u32, avgflooru, addnuw, ARMvshruImm>;
2336
2337multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,
2338                      SDNode unpred_op, Intrinsic pred_int, PatFrag sub_op,
2339                      SDNode shift_op> {
2340  def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2341  defvar Inst = !cast<Instruction>(NAME);
2342
2343  let Predicates = [HasMVEInt] in {
2344    // Unpredicated subtract-and-divide-by-two
2345    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2346                            (i32 VTI.Unsigned))),
2347              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2348
2349    def : Pat<(VTI.Vec (shift_op (sub_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2350              (Inst MQPR:$Qm, MQPR:$Qn)>;
2351
2352
2353    // Predicated subtract-and-divide-by-two
2354    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2355                            (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2356                            (VTI.Vec MQPR:$inactive))),
2357              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2358                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
2359                             (VTI.Vec MQPR:$inactive)))>;
2360  }
2361}
2362
2363multiclass MVE_VHSUB<MVEVectorVTInfo VTI, PatFrag sub_op, SDNode shift_op>
2364  : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, sub_op,
2365                shift_op>;
2366
2367defm MVE_VHSUBs8  : MVE_VHSUB<MVE_v16s8, subnsw, ARMvshrsImm>;
2368defm MVE_VHSUBs16 : MVE_VHSUB<MVE_v8s16, subnsw, ARMvshrsImm>;
2369defm MVE_VHSUBs32 : MVE_VHSUB<MVE_v4s32, subnsw, ARMvshrsImm>;
2370defm MVE_VHSUBu8  : MVE_VHSUB<MVE_v16u8, subnuw, ARMvshruImm>;
2371defm MVE_VHSUBu16 : MVE_VHSUB<MVE_v8u16, subnuw, ARMvshruImm>;
2372defm MVE_VHSUBu32 : MVE_VHSUB<MVE_v4u32, subnuw, ARMvshruImm>;
2373
2374class MVE_VDUP<string suffix, bit B, bit E, bits<2> vecsize, list<dag> pattern=[]>
2375  : MVE_p<(outs MQPR:$Qd), (ins rGPR:$Rt), NoItinerary,
2376          "vdup", suffix, "$Qd, $Rt", vpred_r, "", vecsize, pattern> {
2377  bits<4> Qd;
2378  bits<4> Rt;
2379
2380  let Inst{28} = 0b0;
2381  let Inst{25-23} = 0b101;
2382  let Inst{22} = B;
2383  let Inst{21-20} = 0b10;
2384  let Inst{19-17} = Qd{2-0};
2385  let Inst{16} = 0b0;
2386  let Inst{15-12} = Rt;
2387  let Inst{11-8} = 0b1011;
2388  let Inst{7} = Qd{3};
2389  let Inst{6} = 0b0;
2390  let Inst{5} = E;
2391  let Inst{4-0} = 0b10000;
2392  let validForTailPredication = 1;
2393}
2394
2395def MVE_VDUP32 : MVE_VDUP<"32", 0b0, 0b0, 0b10>;
2396def MVE_VDUP16 : MVE_VDUP<"16", 0b0, 0b1, 0b01>;
2397def MVE_VDUP8  : MVE_VDUP<"8",  0b1, 0b0, 0b00>;
2398
2399let Predicates = [HasMVEInt] in {
2400  def : Pat<(v16i8 (ARMvdup (i32 rGPR:$elem))),
2401            (MVE_VDUP8  rGPR:$elem)>;
2402  def : Pat<(v8i16 (ARMvdup (i32 rGPR:$elem))),
2403            (MVE_VDUP16 rGPR:$elem)>;
2404  def : Pat<(v4i32 (ARMvdup (i32 rGPR:$elem))),
2405            (MVE_VDUP32 rGPR:$elem)>;
2406
2407  def : Pat<(v8f16 (ARMvdup (i32 rGPR:$elem))),
2408            (MVE_VDUP16 rGPR:$elem)>;
2409  def : Pat<(v4f32 (ARMvdup (i32 rGPR:$elem))),
2410            (MVE_VDUP32 rGPR:$elem)>;
2411
2412  // Match a vselect with an ARMvdup as a predicated MVE_VDUP
2413  def : Pat<(v16i8 (vselect (v16i1 VCCR:$pred),
2414                            (v16i8 (ARMvdup (i32 rGPR:$elem))),
2415                            (v16i8 MQPR:$inactive))),
2416            (MVE_VDUP8  rGPR:$elem, ARMVCCThen, (v16i1 VCCR:$pred), zero_reg,
2417                        (v16i8 MQPR:$inactive))>;
2418  def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred),
2419                            (v8i16 (ARMvdup (i32 rGPR:$elem))),
2420                            (v8i16 MQPR:$inactive))),
2421            (MVE_VDUP16 rGPR:$elem, ARMVCCThen, (v8i1 VCCR:$pred), zero_reg,
2422                            (v8i16 MQPR:$inactive))>;
2423  def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred),
2424                            (v4i32 (ARMvdup (i32 rGPR:$elem))),
2425                            (v4i32 MQPR:$inactive))),
2426            (MVE_VDUP32 rGPR:$elem, ARMVCCThen, (v4i1 VCCR:$pred), zero_reg,
2427                            (v4i32 MQPR:$inactive))>;
2428  def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred),
2429                            (v4f32 (ARMvdup (i32 rGPR:$elem))),
2430                            (v4f32 MQPR:$inactive))),
2431            (MVE_VDUP32 rGPR:$elem, ARMVCCThen, (v4i1 VCCR:$pred), zero_reg,
2432                            (v4f32 MQPR:$inactive))>;
2433  def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred),
2434                            (v8f16 (ARMvdup (i32 rGPR:$elem))),
2435                            (v8f16 MQPR:$inactive))),
2436            (MVE_VDUP16 rGPR:$elem, ARMVCCThen, (v8i1 VCCR:$pred), zero_reg,
2437                            (v8f16 MQPR:$inactive))>;
2438}
2439
2440
2441class MVEIntSingleSrc<string iname, string suffix, bits<2> size,
2442                         list<dag> pattern=[]>
2443  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm), NoItinerary,
2444          iname, suffix, "$Qd, $Qm", vpred_r, "", size, pattern> {
2445  bits<4> Qd;
2446  bits<4> Qm;
2447
2448  let Inst{22} = Qd{3};
2449  let Inst{19-18} = size{1-0};
2450  let Inst{15-13} = Qd{2-0};
2451  let Inst{5} = Qm{3};
2452  let Inst{3-1} = Qm{2-0};
2453}
2454
2455class MVE_VCLSCLZ<string iname, string suffix, bits<2> size,
2456                   bit count_zeroes, list<dag> pattern=[]>
2457  : MVEIntSingleSrc<iname, suffix, size, pattern> {
2458
2459  let Inst{28} = 0b1;
2460  let Inst{25-23} = 0b111;
2461  let Inst{21-20} = 0b11;
2462  let Inst{17-16} = 0b00;
2463  let Inst{12-8} = 0b00100;
2464  let Inst{7} = count_zeroes;
2465  let Inst{6} = 0b1;
2466  let Inst{4} = 0b0;
2467  let Inst{0} = 0b0;
2468  let validForTailPredication = 1;
2469}
2470
2471multiclass MVE_VCLSCLZ_p<string opname, bit opcode, MVEVectorVTInfo VTI,
2472                         SDPatternOperator unpred_op> {
2473  def "": MVE_VCLSCLZ<"v"#opname, VTI.Suffix, VTI.Size, opcode>;
2474
2475  defvar Inst     = !cast<Instruction>(NAME);
2476  defvar pred_int = !cast<Intrinsic>("int_arm_mve_"#opname#"_predicated");
2477
2478  let Predicates = [HasMVEInt] in {
2479    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
2480              (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
2481    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
2482                                 (VTI.Vec MQPR:$inactive))),
2483              (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
2484                             (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
2485  }
2486}
2487
2488defm MVE_VCLSs8  : MVE_VCLSCLZ_p<"cls", 0, MVE_v16s8, int_arm_mve_vcls>;
2489defm MVE_VCLSs16 : MVE_VCLSCLZ_p<"cls", 0, MVE_v8s16, int_arm_mve_vcls>;
2490defm MVE_VCLSs32 : MVE_VCLSCLZ_p<"cls", 0, MVE_v4s32, int_arm_mve_vcls>;
2491
2492defm MVE_VCLZs8  : MVE_VCLSCLZ_p<"clz", 1, MVE_v16i8, ctlz>;
2493defm MVE_VCLZs16 : MVE_VCLSCLZ_p<"clz", 1, MVE_v8i16, ctlz>;
2494defm MVE_VCLZs32 : MVE_VCLSCLZ_p<"clz", 1, MVE_v4i32, ctlz>;
2495
2496class MVE_VABSNEG_int<string iname, string suffix, bits<2> size, bit negate,
2497                      bit saturate, list<dag> pattern=[]>
2498  : MVEIntSingleSrc<iname, suffix, size, pattern> {
2499
2500  let Inst{28} = 0b1;
2501  let Inst{25-23} = 0b111;
2502  let Inst{21-20} = 0b11;
2503  let Inst{17} = 0b0;
2504  let Inst{16} = !eq(saturate, 0);
2505  let Inst{12-11} = 0b00;
2506  let Inst{10} = saturate;
2507  let Inst{9-8} = 0b11;
2508  let Inst{7} = negate;
2509  let Inst{6} = 0b1;
2510  let Inst{4} = 0b0;
2511  let Inst{0} = 0b0;
2512  let validForTailPredication = 1;
2513}
2514
2515multiclass MVE_VABSNEG_int_m<string iname, bit negate, bit saturate,
2516                             SDPatternOperator unpred_op, Intrinsic pred_int,
2517                             MVEVectorVTInfo VTI> {
2518  def "" : MVE_VABSNEG_int<iname, VTI.Suffix, VTI.Size, negate, saturate>;
2519  defvar Inst = !cast<Instruction>(NAME);
2520
2521  let Predicates = [HasMVEInt] in {
2522    // VQABS and VQNEG have more difficult isel patterns defined elsewhere
2523    if !not(saturate) then {
2524      def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),
2525                (VTI.Vec (Inst $v))>;
2526    }
2527
2528    def : Pat<(VTI.Vec (pred_int  (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
2529                                  (VTI.Vec MQPR:$inactive))),
2530              (VTI.Vec (Inst $v, ARMVCCThen, $mask, zero_reg, $inactive))>;
2531  }
2532}
2533
2534foreach VTI = [ MVE_v16s8, MVE_v8s16, MVE_v4s32 ] in {
2535  defm "MVE_VABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2536     "vabs",  0, 0, abs,   int_arm_mve_abs_predicated,  VTI>;
2537  defm "MVE_VQABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2538     "vqabs", 0, 1, ?,     int_arm_mve_qabs_predicated, VTI>;
2539  defm "MVE_VNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2540     "vneg",  1, 0, vnegq, int_arm_mve_neg_predicated,  VTI>;
2541  defm "MVE_VQNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2542     "vqneg", 1, 1, ?,     int_arm_mve_qneg_predicated, VTI>;
2543}
2544
2545// int_min/int_max: vector containing INT_MIN/INT_MAX VTI.Size times
2546// zero_vec: v4i32-initialized zero vector, potentially wrapped in a bitconvert
2547multiclass vqabsneg_pattern<MVEVectorVTInfo VTI, dag int_min, dag int_max,
2548                         dag zero_vec,  MVE_VABSNEG_int vqabs_instruction,
2549                         MVE_VABSNEG_int vqneg_instruction> {
2550  let Predicates = [HasMVEInt] in {
2551    // The below tree can be replaced by a vqabs instruction, as it represents
2552    // the following vectorized expression (r being the value in $reg):
2553    // r > 0 ? r : (r == INT_MIN ? INT_MAX : -r)
2554    def : Pat<(VTI.Vec (vselect
2555                      (VTI.Pred (ARMvcmpz (VTI.Vec MQPR:$reg), ARMCCgt)),
2556                      (VTI.Vec MQPR:$reg),
2557                      (VTI.Vec (vselect
2558                                (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2559                                int_max,
2560                                (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))))),
2561            (VTI.Vec (vqabs_instruction (VTI.Vec MQPR:$reg)))>;
2562    // Similarly, this tree represents vqneg, i.e. the following vectorized expression:
2563    // r == INT_MIN ? INT_MAX : -r
2564    def : Pat<(VTI.Vec (vselect
2565                        (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2566                        int_max,
2567                        (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))),
2568               (VTI.Vec (vqneg_instruction (VTI.Vec MQPR:$reg)))>;
2569  }
2570}
2571
2572defm MVE_VQABSNEG_Ps8  : vqabsneg_pattern<MVE_v16i8,
2573                                    (v16i8 (ARMvmovImm (i32 3712))),
2574                                    (v16i8 (ARMvmovImm (i32 3711))),
2575                                    (bitconvert (v4i32 (ARMvmovImm (i32 0)))),
2576                                    MVE_VQABSs8, MVE_VQNEGs8>;
2577defm MVE_VQABSNEG_Ps16 : vqabsneg_pattern<MVE_v8i16,
2578                                    (v8i16 (ARMvmovImm (i32 2688))),
2579                                    (v8i16 (ARMvmvnImm (i32 2688))),
2580                                    (bitconvert (v4i32 (ARMvmovImm (i32 0)))),
2581                                    MVE_VQABSs16, MVE_VQNEGs16>;
2582defm MVE_VQABSNEG_Ps32 : vqabsneg_pattern<MVE_v4i32,
2583                                    (v4i32 (ARMvmovImm (i32 1664))),
2584                                    (v4i32 (ARMvmvnImm (i32 1664))),
2585                                    (ARMvmovImm (i32 0)),
2586                                    MVE_VQABSs32, MVE_VQNEGs32>;
2587
2588class MVE_mod_imm<string iname, string suffix, bits<4> cmode, bit op,
2589                  dag iops, bits<2> vecsize, list<dag> pattern=[]>
2590  : MVE_p<(outs MQPR:$Qd), iops, NoItinerary, iname, suffix, "$Qd, $imm",
2591          vpred_r, "", vecsize, pattern> {
2592  bits<13> imm;
2593  bits<4> Qd;
2594
2595  let Inst{28} = imm{7};
2596  let Inst{25-23} = 0b111;
2597  let Inst{22} = Qd{3};
2598  let Inst{21-19} = 0b000;
2599  let Inst{18-16} = imm{6-4};
2600  let Inst{15-13} = Qd{2-0};
2601  let Inst{12} = 0b0;
2602  let Inst{11-8} = cmode{3-0};
2603  let Inst{7-6} = 0b01;
2604  let Inst{5} = op;
2605  let Inst{4} = 0b1;
2606  let Inst{3-0} = imm{3-0};
2607
2608  let DecoderMethod = "DecodeMVEModImmInstruction";
2609  let validForTailPredication = 1;
2610}
2611
2612let isReMaterializable = 1 in {
2613let isAsCheapAsAMove = 1 in {
2614def MVE_VMOVimmi8  : MVE_mod_imm<"vmov", "i8",  {1,1,1,0}, 0b0, (ins nImmSplatI8:$imm), 0b00>;
2615def MVE_VMOVimmi16 : MVE_mod_imm<"vmov", "i16", {1,0,?,0}, 0b0, (ins nImmSplatI16:$imm), 0b01> {
2616  let Inst{9} = imm{9};
2617}
2618def MVE_VMOVimmi32 : MVE_mod_imm<"vmov", "i32", {?,?,?,?}, 0b0, (ins nImmVMOVI32:$imm), 0b10> {
2619  let Inst{11-8} = imm{11-8};
2620}
2621def MVE_VMOVimmi64 : MVE_mod_imm<"vmov", "i64", {1,1,1,0}, 0b1, (ins nImmSplatI64:$imm), 0b11>;
2622def MVE_VMOVimmf32 : MVE_mod_imm<"vmov", "f32", {1,1,1,1}, 0b0, (ins nImmVMOVF32:$imm), 0b10>;
2623} // let isAsCheapAsAMove = 1
2624
2625def MVE_VMVNimmi16 : MVE_mod_imm<"vmvn", "i16", {1,0,?,0}, 0b1, (ins nImmSplatI16:$imm), 0b01> {
2626  let Inst{9} = imm{9};
2627}
2628def MVE_VMVNimmi32 : MVE_mod_imm<"vmvn", "i32", {?,?,?,?}, 0b1, (ins nImmVMOVI32:$imm), 0b10> {
2629  let Inst{11-8} = imm{11-8};
2630}
2631} // let isReMaterializable = 1
2632
2633let Predicates = [HasMVEInt] in {
2634  def : Pat<(v16i8 (ARMvmovImm timm:$simm)),
2635            (v16i8 (MVE_VMOVimmi8  nImmSplatI8:$simm))>;
2636  def : Pat<(v8i16 (ARMvmovImm timm:$simm)),
2637            (v8i16 (MVE_VMOVimmi16 nImmSplatI16:$simm))>;
2638  def : Pat<(v4i32 (ARMvmovImm timm:$simm)),
2639            (v4i32 (MVE_VMOVimmi32 nImmVMOVI32:$simm))>;
2640  def : Pat<(v2i64 (ARMvmovImm timm:$simm)),
2641            (v2i64 (MVE_VMOVimmi64 nImmSplatI64:$simm))>;
2642
2643  def : Pat<(v8i16 (ARMvmvnImm timm:$simm)),
2644            (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm))>;
2645  def : Pat<(v4i32 (ARMvmvnImm timm:$simm)),
2646            (v4i32 (MVE_VMVNimmi32 nImmVMOVI32:$simm))>;
2647
2648  def : Pat<(v4f32 (ARMvmovFPImm timm:$simm)),
2649            (v4f32 (MVE_VMOVimmf32 nImmVMOVF32:$simm))>;
2650
2651  def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred), (ARMvmvnImm timm:$simm),
2652                            MQPR:$inactive)),
2653            (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm,
2654                            ARMVCCThen, VCCR:$pred, zero_reg, MQPR:$inactive))>;
2655  def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (ARMvmvnImm timm:$simm),
2656                            MQPR:$inactive)),
2657            (v4i32 (MVE_VMVNimmi32 nImmSplatI32:$simm,
2658                            ARMVCCThen, VCCR:$pred, zero_reg, MQPR:$inactive))>;
2659}
2660
2661class MVE_VMINMAXA<string iname, string suffix, bits<2> size,
2662                   bit bit_12, list<dag> pattern=[]>
2663  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
2664          NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",
2665          size, pattern> {
2666  bits<4> Qd;
2667  bits<4> Qm;
2668
2669  let Inst{28} = 0b0;
2670  let Inst{25-23} = 0b100;
2671  let Inst{22} = Qd{3};
2672  let Inst{21-20} = 0b11;
2673  let Inst{19-18} = size;
2674  let Inst{17-16} = 0b11;
2675  let Inst{15-13} = Qd{2-0};
2676  let Inst{12} = bit_12;
2677  let Inst{11-6} = 0b111010;
2678  let Inst{5} = Qm{3};
2679  let Inst{4} = 0b0;
2680  let Inst{3-1} = Qm{2-0};
2681  let Inst{0} = 0b1;
2682  let validForTailPredication = 1;
2683}
2684
2685multiclass MVE_VMINMAXA_m<string iname, MVEVectorVTInfo VTI,
2686                      SDNode unpred_op, Intrinsic pred_int, bit bit_12> {
2687  def "" : MVE_VMINMAXA<iname, VTI.Suffix, VTI.Size, bit_12>;
2688  defvar Inst = !cast<Instruction>(NAME);
2689
2690  let Predicates = [HasMVEInt] in {
2691    // Unpredicated v(min|max)a
2692    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qd), (abs (VTI.Vec MQPR:$Qm)))),
2693              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
2694
2695    // Predicated v(min|max)a
2696    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2697                            (VTI.Pred VCCR:$mask))),
2698              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2699                            ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
2700  }
2701}
2702
2703multiclass MVE_VMINA<MVEVectorVTInfo VTI>
2704  : MVE_VMINMAXA_m<"vmina", VTI, umin, int_arm_mve_vmina_predicated, 0b1>;
2705
2706defm MVE_VMINAs8  : MVE_VMINA<MVE_v16s8>;
2707defm MVE_VMINAs16 : MVE_VMINA<MVE_v8s16>;
2708defm MVE_VMINAs32 : MVE_VMINA<MVE_v4s32>;
2709
2710multiclass MVE_VMAXA<MVEVectorVTInfo VTI>
2711  : MVE_VMINMAXA_m<"vmaxa", VTI, umax, int_arm_mve_vmaxa_predicated, 0b0>;
2712
2713defm MVE_VMAXAs8  : MVE_VMAXA<MVE_v16s8>;
2714defm MVE_VMAXAs16 : MVE_VMAXA<MVE_v8s16>;
2715defm MVE_VMAXAs32 : MVE_VMAXA<MVE_v4s32>;
2716
2717// end of MVE Integer instructions
2718
2719// start of mve_imm_shift instructions
2720
2721def MVE_VSHLC : MVE_p<(outs rGPR:$RdmDest, MQPR:$Qd),
2722                      (ins MQPR:$QdSrc, rGPR:$RdmSrc, long_shift:$imm),
2723                      NoItinerary, "vshlc", "", "$QdSrc, $RdmSrc, $imm",
2724                      vpred_n, "$RdmDest = $RdmSrc,$Qd = $QdSrc", 0b10> {
2725  bits<5> imm;
2726  bits<4> Qd;
2727  bits<4> RdmDest;
2728
2729  let Inst{28} = 0b0;
2730  let Inst{25-23} = 0b101;
2731  let Inst{22} = Qd{3};
2732  let Inst{21} = 0b1;
2733  let Inst{20-16} = imm{4-0};
2734  let Inst{15-13} = Qd{2-0};
2735  let Inst{12-4} = 0b011111100;
2736  let Inst{3-0} = RdmDest{3-0};
2737}
2738
2739class MVE_shift_imm<dag oops, dag iops, string iname, string suffix,
2740                    string ops, vpred_ops vpred, string cstr,
2741                    bits<2> vecsize, list<dag> pattern=[]>
2742  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {
2743  bits<4> Qd;
2744  bits<4> Qm;
2745
2746  let Inst{22} = Qd{3};
2747  let Inst{15-13} = Qd{2-0};
2748  let Inst{5} = Qm{3};
2749  let Inst{3-1} = Qm{2-0};
2750}
2751
2752class MVE_VMOVL<string iname, string suffix, bits<2> sz, bit U, bit top,
2753              list<dag> pattern=[]>
2754  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),
2755                  iname, suffix, "$Qd, $Qm", vpred_r, "",
2756                  sz, pattern> {
2757  let Inst{28} = U;
2758  let Inst{25-23} = 0b101;
2759  let Inst{21} = 0b1;
2760  let Inst{20-19} = sz{1-0};
2761  let Inst{18-16} = 0b000;
2762  let Inst{12} = top;
2763  let Inst{11-6} = 0b111101;
2764  let Inst{4} = 0b0;
2765  let Inst{0} = 0b0;
2766  let doubleWidthResult = 1;
2767}
2768
2769multiclass MVE_VMOVL_m<bit top, string chr, MVEVectorVTInfo OutVTI,
2770                       MVEVectorVTInfo InVTI> {
2771  def "": MVE_VMOVL<"vmovl" # chr, InVTI.Suffix, OutVTI.Size,
2772                    InVTI.Unsigned, top>;
2773  defvar Inst = !cast<Instruction>(NAME);
2774
2775  def : Pat<(OutVTI.Vec (int_arm_mve_vmovl_predicated (InVTI.Vec MQPR:$src),
2776                            (i32 InVTI.Unsigned), (i32 top),
2777                            (OutVTI.Pred VCCR:$pred),
2778                            (OutVTI.Vec MQPR:$inactive))),
2779            (OutVTI.Vec (Inst (InVTI.Vec MQPR:$src), ARMVCCThen,
2780                            (OutVTI.Pred VCCR:$pred), zero_reg,
2781                            (OutVTI.Vec MQPR:$inactive)))>;
2782}
2783
2784defm MVE_VMOVLs8bh  : MVE_VMOVL_m<0, "b", MVE_v8s16, MVE_v16s8>;
2785defm MVE_VMOVLs8th  : MVE_VMOVL_m<1, "t", MVE_v8s16, MVE_v16s8>;
2786defm MVE_VMOVLu8bh  : MVE_VMOVL_m<0, "b", MVE_v8u16, MVE_v16u8>;
2787defm MVE_VMOVLu8th  : MVE_VMOVL_m<1, "t", MVE_v8u16, MVE_v16u8>;
2788defm MVE_VMOVLs16bh : MVE_VMOVL_m<0, "b", MVE_v4s32, MVE_v8s16>;
2789defm MVE_VMOVLs16th : MVE_VMOVL_m<1, "t", MVE_v4s32, MVE_v8s16>;
2790defm MVE_VMOVLu16bh : MVE_VMOVL_m<0, "b", MVE_v4s32, MVE_v8u16>;
2791defm MVE_VMOVLu16th : MVE_VMOVL_m<1, "t", MVE_v4s32, MVE_v8u16>;
2792
2793let Predicates = [HasMVEInt] in {
2794  def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i16),
2795            (MVE_VMOVLs16bh MQPR:$src)>;
2796  def : Pat<(sext_inreg (v8i16 MQPR:$src), v8i8),
2797            (MVE_VMOVLs8bh MQPR:$src)>;
2798  def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i8),
2799            (MVE_VMOVLs16bh (MVE_VMOVLs8bh MQPR:$src))>;
2800
2801  def : Pat<(sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), v8i8),
2802            (MVE_VMOVLs8th MQPR:$src)>;
2803  def : Pat<(sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))), v4i16),
2804            (MVE_VMOVLs16th MQPR:$src)>;
2805
2806  // zext_inreg 8 -> 16
2807  def : Pat<(ARMvbicImm (v8i16 MQPR:$src), (i32 0xAFF)),
2808            (MVE_VMOVLu8bh MQPR:$src)>;
2809  // zext_inreg 16 -> 32
2810  def : Pat<(and (v4i32 MQPR:$src), (v4i32 (ARMvmovImm (i32 0xCFF)))),
2811            (MVE_VMOVLu16bh MQPR:$src)>;
2812  // Same zext_inreg with vrevs, picking the top half
2813  def : Pat<(ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src)))), (i32 0xAFF)),
2814            (MVE_VMOVLu8th MQPR:$src)>;
2815  def : Pat<(and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src)))),
2816                 (v4i32 (ARMvmovImm (i32 0xCFF)))),
2817            (MVE_VMOVLu16th MQPR:$src)>;
2818}
2819
2820
2821class MVE_VSHLL_imm<string iname, string suffix, bit U, bit th,
2822                    Operand immtype, bits<2> vecsize, list<dag> pattern=[]>
2823  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm, immtype:$imm),
2824                  iname, suffix, "$Qd, $Qm, $imm", vpred_r, "", vecsize, pattern> {
2825  let Inst{28} = U;
2826  let Inst{25-23} = 0b101;
2827  let Inst{21} = 0b1;
2828  let Inst{12} = th;
2829  let Inst{11-6} = 0b111101;
2830  let Inst{4} = 0b0;
2831  let Inst{0} = 0b0;
2832
2833  // For the MVE_VSHLL_patterns multiclass to refer to
2834  Operand immediateType = immtype;
2835
2836  let doubleWidthResult = 1;
2837}
2838
2839// The immediate VSHLL instructions accept shift counts from 1 up to
2840// the lane width (8 or 16), but the full-width shifts have an
2841// entirely separate encoding, given below with 'lw' in the name.
2842
2843class MVE_VSHLL_imm8<string iname, string suffix,
2844                     bit U, bit th, list<dag> pattern=[]>
2845  : MVE_VSHLL_imm<iname, suffix, U, th, mve_shift_imm1_7, 0b01, pattern> {
2846  bits<3> imm;
2847  let Inst{20-19} = 0b01;
2848  let Inst{18-16} = imm;
2849}
2850
2851class MVE_VSHLL_imm16<string iname, string suffix,
2852                      bit U, bit th, list<dag> pattern=[]>
2853  : MVE_VSHLL_imm<iname, suffix, U, th, mve_shift_imm1_15, 0b10, pattern> {
2854  bits<4> imm;
2855  let Inst{20} = 0b1;
2856  let Inst{19-16} = imm;
2857}
2858
2859def MVE_VSHLL_imms8bh  : MVE_VSHLL_imm8 <"vshllb", "s8", 0b0, 0b0>;
2860def MVE_VSHLL_imms8th  : MVE_VSHLL_imm8 <"vshllt", "s8", 0b0, 0b1>;
2861def MVE_VSHLL_immu8bh  : MVE_VSHLL_imm8 <"vshllb", "u8", 0b1, 0b0>;
2862def MVE_VSHLL_immu8th  : MVE_VSHLL_imm8 <"vshllt", "u8", 0b1, 0b1>;
2863def MVE_VSHLL_imms16bh : MVE_VSHLL_imm16<"vshllb", "s16", 0b0, 0b0>;
2864def MVE_VSHLL_imms16th : MVE_VSHLL_imm16<"vshllt", "s16", 0b0, 0b1>;
2865def MVE_VSHLL_immu16bh : MVE_VSHLL_imm16<"vshllb", "u16", 0b1, 0b0>;
2866def MVE_VSHLL_immu16th : MVE_VSHLL_imm16<"vshllt", "u16", 0b1, 0b1>;
2867
2868class MVE_VSHLL_by_lane_width<string iname, string suffix, bits<2> size,
2869                              bit U, string ops, list<dag> pattern=[]>
2870  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),
2871                  iname, suffix, ops, vpred_r, "", !if(size, 0b10, 0b01), pattern> {
2872  let Inst{28} = U;
2873  let Inst{25-23} = 0b100;
2874  let Inst{21-20} = 0b11;
2875  let Inst{19-18} = size{1-0};
2876  let Inst{17-16} = 0b01;
2877  let Inst{11-6} = 0b111000;
2878  let Inst{4} = 0b0;
2879  let Inst{0} = 0b1;
2880  let doubleWidthResult = 1;
2881}
2882
2883multiclass MVE_VSHLL_lw<string iname, string suffix, bits<2> sz, bit U,
2884                              string ops, list<dag> pattern=[]> {
2885  def bh : MVE_VSHLL_by_lane_width<iname#"b", suffix, sz, U, ops, pattern> {
2886    let Inst{12} = 0b0;
2887  }
2888  def th : MVE_VSHLL_by_lane_width<iname#"t", suffix, sz, U, ops, pattern> {
2889    let Inst{12} = 0b1;
2890  }
2891}
2892
2893defm MVE_VSHLL_lws8  : MVE_VSHLL_lw<"vshll", "s8",  0b00, 0b0, "$Qd, $Qm, #8">;
2894defm MVE_VSHLL_lws16 : MVE_VSHLL_lw<"vshll", "s16", 0b01, 0b0, "$Qd, $Qm, #16">;
2895defm MVE_VSHLL_lwu8  : MVE_VSHLL_lw<"vshll", "u8",  0b00, 0b1, "$Qd, $Qm, #8">;
2896defm MVE_VSHLL_lwu16 : MVE_VSHLL_lw<"vshll", "u16", 0b01, 0b1, "$Qd, $Qm, #16">;
2897
2898multiclass MVE_VSHLL_patterns<MVEVectorVTInfo VTI, int top> {
2899  defvar suffix     = !strconcat(VTI.Suffix, !if(top, "th", "bh"));
2900  defvar inst_imm   = !cast<MVE_VSHLL_imm>("MVE_VSHLL_imm" # suffix);
2901  defvar inst_lw    = !cast<MVE_VSHLL_by_lane_width>("MVE_VSHLL_lw" # suffix);
2902  defvar unpred_int = int_arm_mve_vshll_imm;
2903  defvar pred_int   = int_arm_mve_vshll_imm_predicated;
2904  defvar imm        = inst_imm.immediateType;
2905
2906  def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), imm:$imm,
2907                                    (i32 VTI.Unsigned), (i32 top))),
2908            (VTI.DblVec (inst_imm   (VTI.Vec MQPR:$src), imm:$imm))>;
2909  def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2910                                    (i32 VTI.Unsigned), (i32 top))),
2911            (VTI.DblVec (inst_lw    (VTI.Vec MQPR:$src)))>;
2912
2913  def : Pat<(VTI.DblVec (pred_int   (VTI.Vec MQPR:$src), imm:$imm,
2914                                    (i32 VTI.Unsigned), (i32 top),
2915                                    (VTI.DblPred VCCR:$mask),
2916                                    (VTI.DblVec MQPR:$inactive))),
2917            (VTI.DblVec (inst_imm   (VTI.Vec MQPR:$src), imm:$imm,
2918                                    ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
2919                                    (VTI.DblVec MQPR:$inactive)))>;
2920  def : Pat<(VTI.DblVec (pred_int   (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2921                                    (i32 VTI.Unsigned), (i32 top),
2922                                    (VTI.DblPred VCCR:$mask),
2923                                    (VTI.DblVec MQPR:$inactive))),
2924            (VTI.DblVec (inst_lw    (VTI.Vec MQPR:$src), ARMVCCThen,
2925                                    (VTI.DblPred VCCR:$mask), zero_reg,
2926                                    (VTI.DblVec MQPR:$inactive)))>;
2927}
2928
2929foreach VTI = [MVE_v16s8, MVE_v8s16, MVE_v16u8, MVE_v8u16] in
2930  foreach top = [0, 1] in
2931    defm : MVE_VSHLL_patterns<VTI, top>;
2932
2933class MVE_shift_imm_partial<Operand imm, string iname, string suffix, bits<2> vecsize>
2934  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$QdSrc, MQPR:$Qm, imm:$imm),
2935                  iname, suffix, "$Qd, $Qm, $imm", vpred_n, "$Qd = $QdSrc", vecsize> {
2936  Operand immediateType = imm;
2937}
2938
2939class MVE_VxSHRN<string iname, string suffix, bit bit_12, bit bit_28,
2940                 Operand imm, bits<2> vecsize>
2941  : MVE_shift_imm_partial<imm, iname, suffix, vecsize> {
2942  bits<5> imm;
2943
2944  let Inst{28} = bit_28;
2945  let Inst{25-23} = 0b101;
2946  let Inst{21} = 0b0;
2947  let Inst{20-16} = imm{4-0};
2948  let Inst{12} = bit_12;
2949  let Inst{11-6} = 0b111111;
2950  let Inst{4} = 0b0;
2951  let Inst{0} = 0b1;
2952  let validForTailPredication = 1;
2953  let retainsPreviousHalfElement = 1;
2954}
2955
2956def MVE_VRSHRNi16bh : MVE_VxSHRN<"vrshrnb", "i16", 0b0, 0b1, shr_imm8, 0b01> {
2957  let Inst{20-19} = 0b01;
2958}
2959def MVE_VRSHRNi16th : MVE_VxSHRN<"vrshrnt", "i16", 0b1, 0b1, shr_imm8, 0b01> {
2960  let Inst{20-19} = 0b01;
2961}
2962def MVE_VRSHRNi32bh : MVE_VxSHRN<"vrshrnb", "i32", 0b0, 0b1, shr_imm16, 0b10> {
2963  let Inst{20} = 0b1;
2964}
2965def MVE_VRSHRNi32th : MVE_VxSHRN<"vrshrnt", "i32", 0b1, 0b1, shr_imm16, 0b10> {
2966  let Inst{20} = 0b1;
2967}
2968
2969def MVE_VSHRNi16bh : MVE_VxSHRN<"vshrnb", "i16", 0b0, 0b0, shr_imm8, 0b01> {
2970  let Inst{20-19} = 0b01;
2971}
2972def MVE_VSHRNi16th : MVE_VxSHRN<"vshrnt", "i16", 0b1, 0b0, shr_imm8, 0b01> {
2973  let Inst{20-19} = 0b01;
2974}
2975def MVE_VSHRNi32bh : MVE_VxSHRN<"vshrnb", "i32", 0b0, 0b0, shr_imm16, 0b10> {
2976  let Inst{20} = 0b1;
2977}
2978def MVE_VSHRNi32th : MVE_VxSHRN<"vshrnt", "i32", 0b1, 0b0, shr_imm16, 0b10> {
2979  let Inst{20} = 0b1;
2980}
2981
2982class MVE_VxQRSHRUN<string iname, string suffix, bit bit_28, bit bit_12,
2983                    Operand imm, bits<2> vecsize>
2984  : MVE_shift_imm_partial<imm, iname, suffix, vecsize> {
2985  bits<5> imm;
2986
2987  let Inst{28} = bit_28;
2988  let Inst{25-23} = 0b101;
2989  let Inst{21} = 0b0;
2990  let Inst{20-16} = imm{4-0};
2991  let Inst{12} = bit_12;
2992  let Inst{11-6} = 0b111111;
2993  let Inst{4} = 0b0;
2994  let Inst{0} = 0b0;
2995  let validForTailPredication = 1;
2996  let retainsPreviousHalfElement = 1;
2997}
2998
2999def MVE_VQRSHRUNs16bh : MVE_VxQRSHRUN<
3000    "vqrshrunb", "s16", 0b1, 0b0, shr_imm8, 0b01> {
3001  let Inst{20-19} = 0b01;
3002}
3003def MVE_VQRSHRUNs16th : MVE_VxQRSHRUN<
3004    "vqrshrunt", "s16", 0b1, 0b1, shr_imm8, 0b01> {
3005  let Inst{20-19} = 0b01;
3006}
3007def MVE_VQRSHRUNs32bh : MVE_VxQRSHRUN<
3008    "vqrshrunb", "s32", 0b1, 0b0, shr_imm16, 0b10> {
3009  let Inst{20} = 0b1;
3010}
3011def MVE_VQRSHRUNs32th : MVE_VxQRSHRUN<
3012    "vqrshrunt", "s32", 0b1, 0b1, shr_imm16, 0b10> {
3013  let Inst{20} = 0b1;
3014}
3015
3016def MVE_VQSHRUNs16bh : MVE_VxQRSHRUN<
3017    "vqshrunb", "s16", 0b0, 0b0, shr_imm8, 0b01> {
3018  let Inst{20-19} = 0b01;
3019}
3020def MVE_VQSHRUNs16th : MVE_VxQRSHRUN<
3021    "vqshrunt", "s16", 0b0, 0b1, shr_imm8, 0b01> {
3022  let Inst{20-19} = 0b01;
3023}
3024def MVE_VQSHRUNs32bh : MVE_VxQRSHRUN<
3025    "vqshrunb", "s32", 0b0, 0b0, shr_imm16, 0b10> {
3026  let Inst{20} = 0b1;
3027}
3028def MVE_VQSHRUNs32th : MVE_VxQRSHRUN<
3029    "vqshrunt", "s32", 0b0, 0b1, shr_imm16, 0b10> {
3030  let Inst{20} = 0b1;
3031}
3032
3033class MVE_VxQRSHRN<string iname, string suffix, bit bit_0, bit bit_12,
3034                   Operand imm, bits<2> vecsize>
3035  : MVE_shift_imm_partial<imm, iname, suffix, vecsize> {
3036  bits<5> imm;
3037
3038  let Inst{25-23} = 0b101;
3039  let Inst{21} = 0b0;
3040  let Inst{20-16} = imm{4-0};
3041  let Inst{12} = bit_12;
3042  let Inst{11-6} = 0b111101;
3043  let Inst{4} = 0b0;
3044  let Inst{0} = bit_0;
3045  let validForTailPredication = 1;
3046  let retainsPreviousHalfElement = 1;
3047}
3048
3049multiclass MVE_VxQRSHRN_types<string iname, bit bit_0, bit bit_12> {
3050  def s16 : MVE_VxQRSHRN<iname, "s16", bit_0, bit_12, shr_imm8, 0b01> {
3051    let Inst{28} = 0b0;
3052    let Inst{20-19} = 0b01;
3053  }
3054  def u16 : MVE_VxQRSHRN<iname, "u16", bit_0, bit_12, shr_imm8, 0b01> {
3055    let Inst{28} = 0b1;
3056    let Inst{20-19} = 0b01;
3057  }
3058  def s32 : MVE_VxQRSHRN<iname, "s32", bit_0, bit_12, shr_imm16, 0b10> {
3059    let Inst{28} = 0b0;
3060    let Inst{20} = 0b1;
3061  }
3062  def u32 : MVE_VxQRSHRN<iname, "u32", bit_0, bit_12, shr_imm16, 0b10> {
3063    let Inst{28} = 0b1;
3064    let Inst{20} = 0b1;
3065  }
3066}
3067
3068defm MVE_VQRSHRNbh : MVE_VxQRSHRN_types<"vqrshrnb", 0b1, 0b0>;
3069defm MVE_VQRSHRNth : MVE_VxQRSHRN_types<"vqrshrnt", 0b1, 0b1>;
3070defm MVE_VQSHRNbh  : MVE_VxQRSHRN_types<"vqshrnb", 0b0, 0b0>;
3071defm MVE_VQSHRNth  : MVE_VxQRSHRN_types<"vqshrnt", 0b0, 0b1>;
3072
3073multiclass MVE_VSHRN_patterns<MVE_shift_imm_partial inst,
3074                              MVEVectorVTInfo OutVTI, MVEVectorVTInfo InVTI,
3075                              bit q, bit r, bit top> {
3076  defvar inparams = (? (OutVTI.Vec MQPR:$QdSrc), (InVTI.Vec MQPR:$Qm),
3077                       (inst.immediateType:$imm), (i32 q), (i32 r),
3078                       (i32 OutVTI.Unsigned), (i32 InVTI.Unsigned), (i32 top));
3079  defvar outparams = (inst (OutVTI.Vec MQPR:$QdSrc), (InVTI.Vec MQPR:$Qm),
3080                           (imm:$imm));
3081
3082  def : Pat<(OutVTI.Vec !setdagop(inparams, int_arm_mve_vshrn)),
3083            (OutVTI.Vec outparams)>;
3084  def : Pat<(OutVTI.Vec !con(inparams, (int_arm_mve_vshrn_predicated
3085                                           (InVTI.Pred VCCR:$pred)))),
3086            (OutVTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred, zero_reg)))>;
3087}
3088
3089defm : MVE_VSHRN_patterns<MVE_VSHRNi16bh,    MVE_v16s8, MVE_v8s16, 0,0,0>;
3090defm : MVE_VSHRN_patterns<MVE_VSHRNi16th,    MVE_v16s8, MVE_v8s16, 0,0,1>;
3091defm : MVE_VSHRN_patterns<MVE_VSHRNi32bh,    MVE_v8s16, MVE_v4s32, 0,0,0>;
3092defm : MVE_VSHRN_patterns<MVE_VSHRNi32th,    MVE_v8s16, MVE_v4s32, 0,0,1>;
3093defm : MVE_VSHRN_patterns<MVE_VSHRNi16bh,    MVE_v16u8, MVE_v8u16, 0,0,0>;
3094defm : MVE_VSHRN_patterns<MVE_VSHRNi16th,    MVE_v16u8, MVE_v8u16, 0,0,1>;
3095defm : MVE_VSHRN_patterns<MVE_VSHRNi32bh,    MVE_v8u16, MVE_v4u32, 0,0,0>;
3096defm : MVE_VSHRN_patterns<MVE_VSHRNi32th,    MVE_v8u16, MVE_v4u32, 0,0,1>;
3097defm : MVE_VSHRN_patterns<MVE_VRSHRNi16bh,   MVE_v16s8, MVE_v8s16, 0,1,0>;
3098defm : MVE_VSHRN_patterns<MVE_VRSHRNi16th,   MVE_v16s8, MVE_v8s16, 0,1,1>;
3099defm : MVE_VSHRN_patterns<MVE_VRSHRNi32bh,   MVE_v8s16, MVE_v4s32, 0,1,0>;
3100defm : MVE_VSHRN_patterns<MVE_VRSHRNi32th,   MVE_v8s16, MVE_v4s32, 0,1,1>;
3101defm : MVE_VSHRN_patterns<MVE_VRSHRNi16bh,   MVE_v16u8, MVE_v8u16, 0,1,0>;
3102defm : MVE_VSHRN_patterns<MVE_VRSHRNi16th,   MVE_v16u8, MVE_v8u16, 0,1,1>;
3103defm : MVE_VSHRN_patterns<MVE_VRSHRNi32bh,   MVE_v8u16, MVE_v4u32, 0,1,0>;
3104defm : MVE_VSHRN_patterns<MVE_VRSHRNi32th,   MVE_v8u16, MVE_v4u32, 0,1,1>;
3105defm : MVE_VSHRN_patterns<MVE_VQSHRNbhs16,   MVE_v16s8, MVE_v8s16, 1,0,0>;
3106defm : MVE_VSHRN_patterns<MVE_VQSHRNths16,   MVE_v16s8, MVE_v8s16, 1,0,1>;
3107defm : MVE_VSHRN_patterns<MVE_VQSHRNbhs32,   MVE_v8s16, MVE_v4s32, 1,0,0>;
3108defm : MVE_VSHRN_patterns<MVE_VQSHRNths32,   MVE_v8s16, MVE_v4s32, 1,0,1>;
3109defm : MVE_VSHRN_patterns<MVE_VQSHRNbhu16,   MVE_v16u8, MVE_v8u16, 1,0,0>;
3110defm : MVE_VSHRN_patterns<MVE_VQSHRNthu16,   MVE_v16u8, MVE_v8u16, 1,0,1>;
3111defm : MVE_VSHRN_patterns<MVE_VQSHRNbhu32,   MVE_v8u16, MVE_v4u32, 1,0,0>;
3112defm : MVE_VSHRN_patterns<MVE_VQSHRNthu32,   MVE_v8u16, MVE_v4u32, 1,0,1>;
3113defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhs16,  MVE_v16s8, MVE_v8s16, 1,1,0>;
3114defm : MVE_VSHRN_patterns<MVE_VQRSHRNths16,  MVE_v16s8, MVE_v8s16, 1,1,1>;
3115defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhs32,  MVE_v8s16, MVE_v4s32, 1,1,0>;
3116defm : MVE_VSHRN_patterns<MVE_VQRSHRNths32,  MVE_v8s16, MVE_v4s32, 1,1,1>;
3117defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhu16,  MVE_v16u8, MVE_v8u16, 1,1,0>;
3118defm : MVE_VSHRN_patterns<MVE_VQRSHRNthu16,  MVE_v16u8, MVE_v8u16, 1,1,1>;
3119defm : MVE_VSHRN_patterns<MVE_VQRSHRNbhu32,  MVE_v8u16, MVE_v4u32, 1,1,0>;
3120defm : MVE_VSHRN_patterns<MVE_VQRSHRNthu32,  MVE_v8u16, MVE_v4u32, 1,1,1>;
3121defm : MVE_VSHRN_patterns<MVE_VQSHRUNs16bh,  MVE_v16u8, MVE_v8s16, 1,0,0>;
3122defm : MVE_VSHRN_patterns<MVE_VQSHRUNs16th,  MVE_v16u8, MVE_v8s16, 1,0,1>;
3123defm : MVE_VSHRN_patterns<MVE_VQSHRUNs32bh,  MVE_v8u16, MVE_v4s32, 1,0,0>;
3124defm : MVE_VSHRN_patterns<MVE_VQSHRUNs32th,  MVE_v8u16, MVE_v4s32, 1,0,1>;
3125defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs16bh, MVE_v16u8, MVE_v8s16, 1,1,0>;
3126defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs16th, MVE_v16u8, MVE_v8s16, 1,1,1>;
3127defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs32bh, MVE_v8u16, MVE_v4s32, 1,1,0>;
3128defm : MVE_VSHRN_patterns<MVE_VQRSHRUNs32th, MVE_v8u16, MVE_v4s32, 1,1,1>;
3129
3130// end of mve_imm_shift instructions
3131
3132// start of mve_shift instructions
3133
3134class MVE_shift_by_vec<string iname, string suffix, bit U,
3135                       bits<2> size, bit bit_4, bit bit_8>
3136  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm, MQPR:$Qn), NoItinerary,
3137           iname, suffix, "$Qd, $Qm, $Qn", vpred_r, "", size, []> {
3138  // Shift instructions which take a vector of shift counts
3139  bits<4> Qd;
3140  bits<4> Qm;
3141  bits<4> Qn;
3142
3143  let Inst{28} = U;
3144  let Inst{25-24} = 0b11;
3145  let Inst{23} = 0b0;
3146  let Inst{22} = Qd{3};
3147  let Inst{21-20} = size;
3148  let Inst{19-17} = Qn{2-0};
3149  let Inst{16} = 0b0;
3150  let Inst{15-13} = Qd{2-0};
3151  let Inst{12-9} = 0b0010;
3152  let Inst{8} = bit_8;
3153  let Inst{7} = Qn{3};
3154  let Inst{6} = 0b1;
3155  let Inst{5} = Qm{3};
3156  let Inst{4} = bit_4;
3157  let Inst{3-1} = Qm{2-0};
3158  let Inst{0} = 0b0;
3159  let validForTailPredication = 1;
3160}
3161
3162multiclass MVE_shift_by_vec_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
3163  def "" : MVE_shift_by_vec<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
3164  defvar Inst = !cast<Instruction>(NAME);
3165
3166  def : Pat<(VTI.Vec (int_arm_mve_vshl_vector
3167                         (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3168                         (i32 q), (i32 r), (i32 VTI.Unsigned))),
3169            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh)))>;
3170
3171  def : Pat<(VTI.Vec (int_arm_mve_vshl_vector_predicated
3172                         (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3173                         (i32 q), (i32 r), (i32 VTI.Unsigned),
3174                         (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
3175            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3176                           ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3177                           (VTI.Vec MQPR:$inactive)))>;
3178}
3179
3180multiclass mve_shift_by_vec_multi<string iname, bit bit_4, bit bit_8> {
3181  defm s8  : MVE_shift_by_vec_p<iname, MVE_v16s8, bit_4, bit_8>;
3182  defm s16 : MVE_shift_by_vec_p<iname, MVE_v8s16, bit_4, bit_8>;
3183  defm s32 : MVE_shift_by_vec_p<iname, MVE_v4s32, bit_4, bit_8>;
3184  defm u8  : MVE_shift_by_vec_p<iname, MVE_v16u8, bit_4, bit_8>;
3185  defm u16 : MVE_shift_by_vec_p<iname, MVE_v8u16, bit_4, bit_8>;
3186  defm u32 : MVE_shift_by_vec_p<iname, MVE_v4u32, bit_4, bit_8>;
3187}
3188
3189defm MVE_VSHL_by_vec   : mve_shift_by_vec_multi<"vshl",   0b0, 0b0>;
3190defm MVE_VQSHL_by_vec  : mve_shift_by_vec_multi<"vqshl",  0b1, 0b0>;
3191defm MVE_VQRSHL_by_vec : mve_shift_by_vec_multi<"vqrshl", 0b1, 0b1>;
3192defm MVE_VRSHL_by_vec  : mve_shift_by_vec_multi<"vrshl",  0b0, 0b1>;
3193
3194let Predicates = [HasMVEInt] in {
3195  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
3196            (v4i32 (MVE_VSHL_by_vecu32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
3197  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
3198            (v8i16 (MVE_VSHL_by_vecu16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
3199  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
3200            (v16i8 (MVE_VSHL_by_vecu8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
3201
3202  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
3203            (v4i32 (MVE_VSHL_by_vecs32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
3204  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
3205            (v8i16 (MVE_VSHL_by_vecs16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
3206  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
3207            (v16i8 (MVE_VSHL_by_vecs8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
3208}
3209
3210class MVE_shift_with_imm<string iname, string suffix, dag oops, dag iops,
3211                         string ops, vpred_ops vpred, string cstr,
3212                         bits<2> vecsize, list<dag> pattern=[]>
3213  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {
3214  bits<4> Qd;
3215  bits<4> Qm;
3216
3217  let Inst{23} = 0b1;
3218  let Inst{22} = Qd{3};
3219  let Inst{15-13} = Qd{2-0};
3220  let Inst{12-11} = 0b00;
3221  let Inst{7-6} = 0b01;
3222  let Inst{5} = Qm{3};
3223  let Inst{4} = 0b1;
3224  let Inst{3-1} = Qm{2-0};
3225  let Inst{0} = 0b0;
3226  let validForTailPredication = 1;
3227
3228  // For the MVE_shift_imm_patterns multiclass to refer to
3229  MVEVectorVTInfo VTI;
3230  Operand immediateType;
3231  Intrinsic unpred_int;
3232  Intrinsic pred_int;
3233  dag unsignedFlag = (?);
3234}
3235
3236class MVE_VSxI_imm<string iname, string suffix, bit bit_8, Operand immType, bits<2> vecsize>
3237  : MVE_shift_with_imm<iname, suffix, (outs MQPR:$Qd),
3238                       (ins MQPR:$Qd_src, MQPR:$Qm, immType:$imm),
3239                       "$Qd, $Qm, $imm", vpred_n, "$Qd = $Qd_src", vecsize> {
3240  bits<6> imm;
3241  let Inst{28} = 0b1;
3242  let Inst{25-24} = 0b11;
3243  let Inst{21-16} = imm;
3244  let Inst{10-9} = 0b10;
3245  let Inst{8} = bit_8;
3246  let validForTailPredication = 1;
3247
3248  Operand immediateType = immType;
3249}
3250
3251def MVE_VSRIimm8 : MVE_VSxI_imm<"vsri", "8", 0b0, shr_imm8, 0b00> {
3252  let Inst{21-19} = 0b001;
3253}
3254
3255def MVE_VSRIimm16 : MVE_VSxI_imm<"vsri", "16", 0b0, shr_imm16, 0b01> {
3256  let Inst{21-20} = 0b01;
3257}
3258
3259def MVE_VSRIimm32 : MVE_VSxI_imm<"vsri", "32", 0b0, shr_imm32, 0b10> {
3260  let Inst{21} = 0b1;
3261}
3262
3263def MVE_VSLIimm8 : MVE_VSxI_imm<"vsli", "8", 0b1, imm0_7, 0b00> {
3264  let Inst{21-19} = 0b001;
3265}
3266
3267def MVE_VSLIimm16 : MVE_VSxI_imm<"vsli", "16", 0b1, imm0_15, 0b01> {
3268  let Inst{21-20} = 0b01;
3269}
3270
3271def MVE_VSLIimm32 : MVE_VSxI_imm<"vsli", "32", 0b1,imm0_31, 0b10> {
3272  let Inst{21} = 0b1;
3273}
3274
3275multiclass MVE_VSxI_patterns<MVE_VSxI_imm inst, string name,
3276                              MVEVectorVTInfo VTI> {
3277  defvar inparams = (? (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3278                       (inst.immediateType:$imm));
3279  defvar outparams = (inst (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3280                           (inst.immediateType:$imm));
3281  defvar unpred_int = !cast<Intrinsic>("int_arm_mve_" # name);
3282  defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # name # "_predicated");
3283
3284  def : Pat<(VTI.Vec !setdagop(inparams, unpred_int)),
3285            (VTI.Vec outparams)>;
3286  def : Pat<(VTI.Vec !con(inparams, (pred_int (VTI.Pred VCCR:$pred)))),
3287            (VTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred, zero_reg)))>;
3288}
3289
3290defm : MVE_VSxI_patterns<MVE_VSLIimm8,  "vsli", MVE_v16i8>;
3291defm : MVE_VSxI_patterns<MVE_VSLIimm16, "vsli", MVE_v8i16>;
3292defm : MVE_VSxI_patterns<MVE_VSLIimm32, "vsli", MVE_v4i32>;
3293defm : MVE_VSxI_patterns<MVE_VSRIimm8,  "vsri", MVE_v16i8>;
3294defm : MVE_VSxI_patterns<MVE_VSRIimm16, "vsri", MVE_v8i16>;
3295defm : MVE_VSxI_patterns<MVE_VSRIimm32, "vsri", MVE_v4i32>;
3296
3297class MVE_VQSHL_imm<MVEVectorVTInfo VTI_, Operand immType>
3298  : MVE_shift_with_imm<"vqshl", VTI_.Suffix, (outs MQPR:$Qd),
3299                       (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",
3300                       vpred_r, "", VTI_.Size> {
3301  bits<6> imm;
3302
3303  let Inst{28} = VTI_.Unsigned;
3304  let Inst{25-24} = 0b11;
3305  let Inst{21-16} = imm;
3306  let Inst{10-8} = 0b111;
3307
3308  let VTI = VTI_;
3309  let immediateType = immType;
3310  let unsignedFlag = (? (i32 VTI.Unsigned));
3311}
3312
3313let unpred_int = int_arm_mve_vqshl_imm,
3314    pred_int = int_arm_mve_vqshl_imm_predicated in {
3315  def MVE_VQSHLimms8 : MVE_VQSHL_imm<MVE_v16s8, imm0_7> {
3316    let Inst{21-19} = 0b001;
3317  }
3318  def MVE_VQSHLimmu8 : MVE_VQSHL_imm<MVE_v16u8, imm0_7> {
3319    let Inst{21-19} = 0b001;
3320  }
3321
3322  def MVE_VQSHLimms16 : MVE_VQSHL_imm<MVE_v8s16, imm0_15> {
3323    let Inst{21-20} = 0b01;
3324  }
3325  def MVE_VQSHLimmu16 : MVE_VQSHL_imm<MVE_v8u16, imm0_15> {
3326    let Inst{21-20} = 0b01;
3327  }
3328
3329  def MVE_VQSHLimms32 : MVE_VQSHL_imm<MVE_v4s32, imm0_31> {
3330    let Inst{21} = 0b1;
3331  }
3332  def MVE_VQSHLimmu32 : MVE_VQSHL_imm<MVE_v4u32, imm0_31> {
3333    let Inst{21} = 0b1;
3334  }
3335}
3336
3337class MVE_VQSHLU_imm<MVEVectorVTInfo VTI_, Operand immType>
3338  : MVE_shift_with_imm<"vqshlu", VTI_.Suffix, (outs MQPR:$Qd),
3339                       (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",
3340                       vpred_r, "", VTI_.Size> {
3341  bits<6> imm;
3342
3343  let Inst{28} = 0b1;
3344  let Inst{25-24} = 0b11;
3345  let Inst{21-16} = imm;
3346  let Inst{10-8} = 0b110;
3347
3348  let VTI = VTI_;
3349  let immediateType = immType;
3350}
3351
3352let unpred_int = int_arm_mve_vqshlu_imm,
3353    pred_int = int_arm_mve_vqshlu_imm_predicated in {
3354  def MVE_VQSHLU_imms8 : MVE_VQSHLU_imm<MVE_v16s8, imm0_7> {
3355    let Inst{21-19} = 0b001;
3356  }
3357
3358  def MVE_VQSHLU_imms16 : MVE_VQSHLU_imm<MVE_v8s16, imm0_15> {
3359    let Inst{21-20} = 0b01;
3360  }
3361
3362  def MVE_VQSHLU_imms32 : MVE_VQSHLU_imm<MVE_v4s32, imm0_31> {
3363    let Inst{21} = 0b1;
3364  }
3365}
3366
3367class MVE_VRSHR_imm<MVEVectorVTInfo VTI_, Operand immType>
3368  : MVE_shift_with_imm<"vrshr", VTI_.Suffix, (outs MQPR:$Qd),
3369                       (ins MQPR:$Qm, immType:$imm), "$Qd, $Qm, $imm",
3370                       vpred_r, "", VTI_.Size> {
3371  bits<6> imm;
3372
3373  let Inst{28} = VTI_.Unsigned;
3374  let Inst{25-24} = 0b11;
3375  let Inst{21-16} = imm;
3376  let Inst{10-8} = 0b010;
3377
3378  let VTI = VTI_;
3379  let immediateType = immType;
3380  let unsignedFlag = (? (i32 VTI.Unsigned));
3381}
3382
3383let unpred_int = int_arm_mve_vrshr_imm,
3384    pred_int = int_arm_mve_vrshr_imm_predicated in {
3385  def MVE_VRSHR_imms8 : MVE_VRSHR_imm<MVE_v16s8, shr_imm8> {
3386    let Inst{21-19} = 0b001;
3387  }
3388
3389  def MVE_VRSHR_immu8 : MVE_VRSHR_imm<MVE_v16u8, shr_imm8> {
3390    let Inst{21-19} = 0b001;
3391  }
3392
3393  def MVE_VRSHR_imms16 : MVE_VRSHR_imm<MVE_v8s16, shr_imm16> {
3394    let Inst{21-20} = 0b01;
3395  }
3396
3397  def MVE_VRSHR_immu16 : MVE_VRSHR_imm<MVE_v8u16, shr_imm16> {
3398    let Inst{21-20} = 0b01;
3399  }
3400
3401  def MVE_VRSHR_imms32 : MVE_VRSHR_imm<MVE_v4s32, shr_imm32> {
3402    let Inst{21} = 0b1;
3403  }
3404
3405  def MVE_VRSHR_immu32 : MVE_VRSHR_imm<MVE_v4u32, shr_imm32> {
3406    let Inst{21} = 0b1;
3407  }
3408}
3409
3410multiclass MVE_shift_imm_patterns<MVE_shift_with_imm inst> {
3411  def : Pat<(inst.VTI.Vec !con((inst.unpred_int (inst.VTI.Vec MQPR:$src),
3412                                                inst.immediateType:$imm),
3413                               inst.unsignedFlag)),
3414            (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3415                                inst.immediateType:$imm))>;
3416
3417  def : Pat<(inst.VTI.Vec !con((inst.pred_int (inst.VTI.Vec MQPR:$src),
3418                                              inst.immediateType:$imm),
3419                               inst.unsignedFlag,
3420                               (? (inst.VTI.Pred VCCR:$mask),
3421                                  (inst.VTI.Vec MQPR:$inactive)))),
3422            (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3423                                inst.immediateType:$imm,
3424                                ARMVCCThen, (inst.VTI.Pred VCCR:$mask), zero_reg,
3425                                (inst.VTI.Vec MQPR:$inactive)))>;
3426}
3427
3428defm : MVE_shift_imm_patterns<MVE_VQSHLimms8>;
3429defm : MVE_shift_imm_patterns<MVE_VQSHLimmu8>;
3430defm : MVE_shift_imm_patterns<MVE_VQSHLimms16>;
3431defm : MVE_shift_imm_patterns<MVE_VQSHLimmu16>;
3432defm : MVE_shift_imm_patterns<MVE_VQSHLimms32>;
3433defm : MVE_shift_imm_patterns<MVE_VQSHLimmu32>;
3434defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms8>;
3435defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms16>;
3436defm : MVE_shift_imm_patterns<MVE_VQSHLU_imms32>;
3437defm : MVE_shift_imm_patterns<MVE_VRSHR_imms8>;
3438defm : MVE_shift_imm_patterns<MVE_VRSHR_immu8>;
3439defm : MVE_shift_imm_patterns<MVE_VRSHR_imms16>;
3440defm : MVE_shift_imm_patterns<MVE_VRSHR_immu16>;
3441defm : MVE_shift_imm_patterns<MVE_VRSHR_imms32>;
3442defm : MVE_shift_imm_patterns<MVE_VRSHR_immu32>;
3443
3444class MVE_VSHR_imm<string suffix, dag imm, bits<2> vecsize>
3445  : MVE_shift_with_imm<"vshr", suffix, (outs MQPR:$Qd),
3446                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
3447                       vpred_r, "", vecsize> {
3448  bits<6> imm;
3449
3450  let Inst{25-24} = 0b11;
3451  let Inst{21-16} = imm;
3452  let Inst{10-8} = 0b000;
3453}
3454
3455def MVE_VSHR_imms8 : MVE_VSHR_imm<"s8", (ins shr_imm8:$imm), 0b00> {
3456  let Inst{28} = 0b0;
3457  let Inst{21-19} = 0b001;
3458}
3459
3460def MVE_VSHR_immu8 : MVE_VSHR_imm<"u8", (ins shr_imm8:$imm), 0b00> {
3461  let Inst{28} = 0b1;
3462  let Inst{21-19} = 0b001;
3463}
3464
3465def MVE_VSHR_imms16 : MVE_VSHR_imm<"s16", (ins shr_imm16:$imm), 0b01> {
3466  let Inst{28} = 0b0;
3467  let Inst{21-20} = 0b01;
3468}
3469
3470def MVE_VSHR_immu16 : MVE_VSHR_imm<"u16", (ins shr_imm16:$imm), 0b01> {
3471  let Inst{28} = 0b1;
3472  let Inst{21-20} = 0b01;
3473}
3474
3475def MVE_VSHR_imms32 : MVE_VSHR_imm<"s32", (ins shr_imm32:$imm), 0b10> {
3476  let Inst{28} = 0b0;
3477  let Inst{21} = 0b1;
3478}
3479
3480def MVE_VSHR_immu32 : MVE_VSHR_imm<"u32", (ins shr_imm32:$imm), 0b10> {
3481  let Inst{28} = 0b1;
3482  let Inst{21} = 0b1;
3483}
3484
3485class MVE_VSHL_imm<string suffix, dag imm, bits<2> vecsize>
3486  : MVE_shift_with_imm<"vshl", suffix, (outs MQPR:$Qd),
3487                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
3488                       vpred_r, "", vecsize> {
3489  bits<6> imm;
3490
3491  let Inst{28} = 0b0;
3492  let Inst{25-24} = 0b11;
3493  let Inst{21-16} = imm;
3494  let Inst{10-8} = 0b101;
3495}
3496
3497def MVE_VSHL_immi8 : MVE_VSHL_imm<"i8", (ins imm0_7:$imm), 0b00> {
3498  let Inst{21-19} = 0b001;
3499}
3500
3501def MVE_VSHL_immi16 : MVE_VSHL_imm<"i16", (ins imm0_15:$imm), 0b01> {
3502  let Inst{21-20} = 0b01;
3503}
3504
3505def MVE_VSHL_immi32 : MVE_VSHL_imm<"i32", (ins imm0_31:$imm), 0b10> {
3506  let Inst{21} = 0b1;
3507}
3508
3509multiclass MVE_immediate_shift_patterns_inner<
3510    MVEVectorVTInfo VTI, Operand imm_operand_type, SDNode unpred_op,
3511    Intrinsic pred_int, Instruction inst, list<int> unsignedFlag = []> {
3512
3513  def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src), imm_operand_type:$imm)),
3514            (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm))>;
3515
3516  def : Pat<(VTI.Vec !con((pred_int (VTI.Vec MQPR:$src), imm_operand_type:$imm),
3517                          !dag(pred_int, unsignedFlag, ?),
3518                          (pred_int (VTI.Pred VCCR:$mask),
3519                                   (VTI.Vec MQPR:$inactive)))),
3520            (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm,
3521                           ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3522                           (VTI.Vec MQPR:$inactive)))>;
3523}
3524
3525multiclass MVE_immediate_shift_patterns<MVEVectorVTInfo VTI,
3526                                        Operand imm_operand_type> {
3527  defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3528      ARMvshlImm, int_arm_mve_shl_imm_predicated,
3529      !cast<Instruction>("MVE_VSHL_immi" # VTI.BitsSuffix)>;
3530  defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3531      ARMvshruImm, int_arm_mve_shr_imm_predicated,
3532      !cast<Instruction>("MVE_VSHR_immu" # VTI.BitsSuffix), [1]>;
3533  defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3534      ARMvshrsImm, int_arm_mve_shr_imm_predicated,
3535      !cast<Instruction>("MVE_VSHR_imms" # VTI.BitsSuffix), [0]>;
3536}
3537
3538let Predicates = [HasMVEInt] in {
3539  defm : MVE_immediate_shift_patterns<MVE_v16i8, imm0_7>;
3540  defm : MVE_immediate_shift_patterns<MVE_v8i16, imm0_15>;
3541  defm : MVE_immediate_shift_patterns<MVE_v4i32, imm0_31>;
3542}
3543
3544// end of mve_shift instructions
3545
3546// start of MVE Floating Point instructions
3547
3548class MVE_float<string iname, string suffix, dag oops, dag iops, string ops,
3549                vpred_ops vpred, string cstr, bits<2> vecsize, list<dag> pattern=[]>
3550  : MVE_f<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {
3551  bits<4> Qm;
3552
3553  let Inst{12} = 0b0;
3554  let Inst{6} = 0b1;
3555  let Inst{5} = Qm{3};
3556  let Inst{3-1} = Qm{2-0};
3557  let Inst{0} = 0b0;
3558}
3559
3560class MVE_VRINT<string rmode, bits<3> op, string suffix, bits<2> size,
3561                list<dag> pattern=[]>
3562  : MVE_float<!strconcat("vrint", rmode), suffix, (outs MQPR:$Qd),
3563              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {
3564  bits<4> Qd;
3565
3566  let Inst{28} = 0b1;
3567  let Inst{25-23} = 0b111;
3568  let Inst{22} = Qd{3};
3569  let Inst{21-20} = 0b11;
3570  let Inst{19-18} = size;
3571  let Inst{17-16} = 0b10;
3572  let Inst{15-13} = Qd{2-0};
3573  let Inst{11-10} = 0b01;
3574  let Inst{9-7} = op{2-0};
3575  let Inst{4} = 0b0;
3576  let validForTailPredication = 1;
3577
3578}
3579
3580multiclass MVE_VRINT_m<MVEVectorVTInfo VTI, string suffix, bits<3> opcode,
3581                       SDPatternOperator unpred_op> {
3582  def "": MVE_VRINT<suffix, opcode, VTI.Suffix, VTI.Size>;
3583  defvar Inst = !cast<Instruction>(NAME);
3584  defvar pred_int = !cast<Intrinsic>("int_arm_mve_vrint"#suffix#"_predicated");
3585
3586  let Predicates = [HasMVEFloat] in {
3587    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
3588              (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
3589    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
3590                                 (VTI.Vec MQPR:$inactive))),
3591              (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
3592                             (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
3593  }
3594}
3595
3596multiclass MVE_VRINT_ops<MVEVectorVTInfo VTI> {
3597  defm N : MVE_VRINT_m<VTI, "n", 0b000, int_arm_mve_vrintn>;
3598  defm X : MVE_VRINT_m<VTI, "x", 0b001, frint>;
3599  defm A : MVE_VRINT_m<VTI, "a", 0b010, fround>;
3600  defm Z : MVE_VRINT_m<VTI, "z", 0b011, ftrunc>;
3601  defm M : MVE_VRINT_m<VTI, "m", 0b101, ffloor>;
3602  defm P : MVE_VRINT_m<VTI, "p", 0b111, fceil>;
3603}
3604
3605defm MVE_VRINTf16 : MVE_VRINT_ops<MVE_v8f16>;
3606defm MVE_VRINTf32 : MVE_VRINT_ops<MVE_v4f32>;
3607
3608class MVEFloatArithNeon<string iname, string suffix, bit size,
3609                           dag oops, dag iops, string ops,
3610                           vpred_ops vpred, string cstr, bits<2> vecsize, list<dag> pattern=[]>
3611  : MVE_float<iname, suffix, oops, iops, ops, vpred, cstr, vecsize, pattern> {
3612  let Inst{20} = size;
3613  let Inst{16} = 0b0;
3614}
3615
3616class MVE_VMUL_fp<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
3617  : MVEFloatArithNeon<iname, suffix, size{0}, (outs MQPR:$Qd),
3618                      (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm", vpred_r, "",
3619                      size, pattern> {
3620  bits<4> Qd;
3621  bits<4> Qn;
3622
3623  let Inst{28} = 0b1;
3624  let Inst{25-23} = 0b110;
3625  let Inst{22} = Qd{3};
3626  let Inst{21} = 0b0;
3627  let Inst{19-17} = Qn{2-0};
3628  let Inst{15-13} = Qd{2-0};
3629  let Inst{12-8} = 0b01101;
3630  let Inst{7} = Qn{3};
3631  let Inst{4} = 0b1;
3632  let validForTailPredication = 1;
3633}
3634
3635multiclass MVE_VMULT_fp_m<string iname, MVEVectorVTInfo VTI, SDNode Op,
3636                          Intrinsic PredInt, SDPatternOperator IdentityVec> {
3637  def "" : MVE_VMUL_fp<iname, VTI.Suffix, VTI.Size>;
3638  defvar Inst = !cast<Instruction>(NAME);
3639
3640  let Predicates = [HasMVEFloat] in {
3641    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), IdentityVec>;
3642  }
3643}
3644
3645multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>
3646  : MVE_VMULT_fp_m<"vmul", VTI, fmul, int_arm_mve_mul_predicated, IdentityVec>;
3647
3648def ARMimmOneF: PatLeaf<(bitconvert (v4f32 (ARMvmovFPImm (i32 112))))>; // 1.0 float
3649def ARMimmOneH: PatLeaf<(bitconvert (v8i16 (ARMvmovImm (i32 2620))))>; // 1.0 half
3650
3651defm MVE_VMULf32 : MVE_VMUL_fp_m<MVE_v4f32, ARMimmOneF>;
3652defm MVE_VMULf16 : MVE_VMUL_fp_m<MVE_v8f16, ARMimmOneH>;
3653
3654class MVE_VCMLA<string suffix, bits<2> size>
3655  : MVEFloatArithNeon<"vcmla", suffix, size{1}, (outs MQPR:$Qd),
3656                         (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
3657                         "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", size, []> {
3658  bits<4> Qd;
3659  bits<4> Qn;
3660  bits<2> rot;
3661
3662  let Inst{28} = 0b1;
3663  let Inst{25} = 0b0;
3664  let Inst{24-23} = rot;
3665  let Inst{22} = Qd{3};
3666  let Inst{21} = 0b1;
3667  let Inst{19-17} = Qn{2-0};
3668  let Inst{15-13} = Qd{2-0};
3669  let Inst{12-8} = 0b01000;
3670  let Inst{7} = Qn{3};
3671  let Inst{4} = 0b0;
3672}
3673
3674multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI> {
3675  def "" : MVE_VCMLA<VTI.Suffix, VTI.Size>;
3676  defvar Inst = !cast<Instruction>(NAME);
3677
3678  let Predicates = [HasMVEFloat] in {
3679    def : Pat<(VTI.Vec (int_arm_mve_vcmlaq
3680                            imm:$rot, (VTI.Vec MQPR:$Qd_src),
3681                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3682              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
3683                             (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3684                             imm:$rot))>;
3685
3686    def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated
3687                            imm:$rot, (VTI.Vec MQPR:$Qd_src),
3688                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3689                            (VTI.Pred VCCR:$mask))),
3690              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),
3691                             (VTI.Vec MQPR:$Qm), imm:$rot,
3692                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
3693
3694  }
3695}
3696
3697defm MVE_VCMLAf16 : MVE_VCMLA_m<MVE_v8f16>;
3698defm MVE_VCMLAf32 : MVE_VCMLA_m<MVE_v4f32>;
3699
3700class MVE_VADDSUBFMA_fp<string iname, string suffix, bits<2> size, bit bit_4,
3701                        bit bit_8, bit bit_21, dag iops=(ins),
3702                        vpred_ops vpred=vpred_r, string cstr="",
3703                        list<dag> pattern=[]>
3704  : MVEFloatArithNeon<iname, suffix, size{0}, (outs MQPR:$Qd),
3705                      !con(iops, (ins MQPR:$Qn, MQPR:$Qm)), "$Qd, $Qn, $Qm",
3706                      vpred, cstr, size, pattern> {
3707  bits<4> Qd;
3708  bits<4> Qn;
3709
3710  let Inst{28} = 0b0;
3711  let Inst{25-23} = 0b110;
3712  let Inst{22} = Qd{3};
3713  let Inst{21} = bit_21;
3714  let Inst{19-17} = Qn{2-0};
3715  let Inst{15-13} = Qd{2-0};
3716  let Inst{11-9} = 0b110;
3717  let Inst{8} = bit_8;
3718  let Inst{7} = Qn{3};
3719  let Inst{4} = bit_4;
3720  let validForTailPredication = 1;
3721}
3722
3723multiclass MVE_VFMA_fp_multi<string iname, bit fms, MVEVectorVTInfo VTI> {
3724  def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size, 0b1, 0b0, fms,
3725                             (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
3726  defvar Inst = !cast<Instruction>(NAME);
3727  defvar pred_int = int_arm_mve_fma_predicated;
3728  defvar m1   = (VTI.Vec MQPR:$m1);
3729  defvar m2   = (VTI.Vec MQPR:$m2);
3730  defvar add  = (VTI.Vec MQPR:$add);
3731  defvar pred = (VTI.Pred VCCR:$pred);
3732
3733  let Predicates = [HasMVEFloat] in {
3734    if fms then {
3735      def : Pat<(VTI.Vec (fma (fneg m1), m2, add)),
3736                (Inst $add, $m1, $m2)>;
3737      def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
3738                                  (VTI.Vec (fma (fneg m1), m2, add)),
3739                                  add)),
3740                (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;
3741      def : Pat<(VTI.Vec (pred_int (fneg m1), m2, add, pred)),
3742                (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;
3743      def : Pat<(VTI.Vec (pred_int m1, (fneg m2), add, pred)),
3744                (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;
3745    } else {
3746      def : Pat<(VTI.Vec (fma m1, m2, add)),
3747                (Inst $add, $m1, $m2)>;
3748      def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
3749                                  (VTI.Vec (fma m1, m2, add)),
3750                                  add)),
3751                (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;
3752      def : Pat<(VTI.Vec (pred_int m1, m2, add, pred)),
3753                (Inst $add, $m1, $m2, ARMVCCThen, $pred, zero_reg)>;
3754    }
3755  }
3756}
3757
3758defm MVE_VFMAf32 : MVE_VFMA_fp_multi<"vfma", 0, MVE_v4f32>;
3759defm MVE_VFMAf16 : MVE_VFMA_fp_multi<"vfma", 0, MVE_v8f16>;
3760defm MVE_VFMSf32 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v4f32>;
3761defm MVE_VFMSf16 : MVE_VFMA_fp_multi<"vfms", 1, MVE_v8f16>;
3762
3763multiclass MVE_VADDSUB_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
3764                            SDNode Op, Intrinsic PredInt, SDPatternOperator IdentityVec> {
3765  def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size, 0, 1, bit_21> {
3766    let validForTailPredication = 1;
3767  }
3768  defvar Inst = !cast<Instruction>(NAME);
3769
3770  let Predicates = [HasMVEFloat] in {
3771    defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), IdentityVec>;
3772  }
3773}
3774
3775multiclass MVE_VADD_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>
3776  : MVE_VADDSUB_fp_m<"vadd", 0, VTI, fadd, int_arm_mve_add_predicated, IdentityVec>;
3777multiclass MVE_VSUB_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>
3778  : MVE_VADDSUB_fp_m<"vsub", 1, VTI, fsub, int_arm_mve_sub_predicated, IdentityVec>;
3779
3780def ARMimmMinusZeroF: PatLeaf<(bitconvert (v4i32 (ARMvmovImm (i32 1664))))>; // -0.0 float
3781def ARMimmMinusZeroH: PatLeaf<(bitconvert (v8i16 (ARMvmovImm (i32 2688))))>; // -0.0 half
3782
3783defm MVE_VADDf32 : MVE_VADD_fp_m<MVE_v4f32, ARMimmMinusZeroF>;
3784defm MVE_VADDf16 : MVE_VADD_fp_m<MVE_v8f16, ARMimmMinusZeroH>;
3785
3786defm MVE_VSUBf32 : MVE_VSUB_fp_m<MVE_v4f32, ARMimmAllZerosV>;
3787defm MVE_VSUBf16 : MVE_VSUB_fp_m<MVE_v8f16, ARMimmAllZerosV>;
3788
3789class MVE_VCADD<string suffix, bits<2> size, string cstr="">
3790  : MVEFloatArithNeon<"vcadd", suffix, size{1}, (outs MQPR:$Qd),
3791                         (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
3792                         "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, size, []> {
3793  bits<4> Qd;
3794  bits<4> Qn;
3795  bit rot;
3796
3797  let Inst{28} = 0b1;
3798  let Inst{25} = 0b0;
3799  let Inst{24} = rot;
3800  let Inst{23} = 0b1;
3801  let Inst{22} = Qd{3};
3802  let Inst{21} = 0b0;
3803  let Inst{19-17} = Qn{2-0};
3804  let Inst{15-13} = Qd{2-0};
3805  let Inst{12-8} = 0b01000;
3806  let Inst{7} = Qn{3};
3807  let Inst{4} = 0b0;
3808}
3809
3810multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, string cstr=""> {
3811  def "" : MVE_VCADD<VTI.Suffix, VTI.Size, cstr>;
3812  defvar Inst = !cast<Instruction>(NAME);
3813
3814  let Predicates = [HasMVEFloat] in {
3815    def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),
3816                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3817              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3818                             imm:$rot))>;
3819
3820    def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),
3821                            imm:$rot, (VTI.Vec MQPR:$inactive),
3822                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3823                            (VTI.Pred VCCR:$mask))),
3824              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3825                             imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3826                             (VTI.Vec MQPR:$inactive)))>;
3827
3828  }
3829}
3830
3831defm MVE_VCADDf16 : MVE_VCADD_m<MVE_v8f16>;
3832defm MVE_VCADDf32 : MVE_VCADD_m<MVE_v4f32, "@earlyclobber $Qd">;
3833
3834class MVE_VABD_fp<string suffix, bits<2> size>
3835  : MVE_float<"vabd", suffix, (outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
3836              "$Qd, $Qn, $Qm", vpred_r, "", size> {
3837  bits<4> Qd;
3838  bits<4> Qn;
3839
3840  let Inst{28} = 0b1;
3841  let Inst{25-23} = 0b110;
3842  let Inst{22} = Qd{3};
3843  let Inst{21} = 0b1;
3844  let Inst{20} = size{0};
3845  let Inst{19-17} = Qn{2-0};
3846  let Inst{16} = 0b0;
3847  let Inst{15-13} = Qd{2-0};
3848  let Inst{11-8} = 0b1101;
3849  let Inst{7} = Qn{3};
3850  let Inst{4} = 0b0;
3851  let validForTailPredication = 1;
3852}
3853
3854multiclass MVE_VABDT_fp_m<MVEVectorVTInfo VTI,
3855                            Intrinsic unpred_int, Intrinsic pred_int> {
3856  def "" : MVE_VABD_fp<VTI.Suffix, VTI.Size>;
3857  defvar Inst = !cast<Instruction>(NAME);
3858
3859  let Predicates = [HasMVEFloat] in {
3860    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3861                            (i32 0))),
3862              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
3863    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3864                            (i32 0), (VTI.Pred VCCR:$mask),
3865                            (VTI.Vec MQPR:$inactive))),
3866              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3867                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3868                             (VTI.Vec MQPR:$inactive)))>;
3869  }
3870}
3871
3872multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>
3873  : MVE_VABDT_fp_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
3874
3875defm MVE_VABDf32 : MVE_VABD_fp_m<MVE_v4f32>;
3876defm MVE_VABDf16 : MVE_VABD_fp_m<MVE_v8f16>;
3877
3878let Predicates = [HasMVEFloat] in {
3879  def : Pat<(v8f16 (fabs (fsub (v8f16 MQPR:$Qm), (v8f16 MQPR:$Qn)))),
3880            (MVE_VABDf16 MQPR:$Qm, MQPR:$Qn)>;
3881  def : Pat<(v4f32 (fabs (fsub (v4f32 MQPR:$Qm), (v4f32 MQPR:$Qn)))),
3882            (MVE_VABDf32 MQPR:$Qm, MQPR:$Qn)>;
3883}
3884
3885class MVE_VCVT_fix<string suffix, bit fsi, bit U, bit op,
3886                   Operand imm_operand_type>
3887  : MVE_float<"vcvt", suffix,
3888              (outs MQPR:$Qd), (ins MQPR:$Qm, imm_operand_type:$imm6),
3889              "$Qd, $Qm, $imm6", vpred_r, "", !if(fsi, 0b10, 0b01), []> {
3890  bits<4> Qd;
3891  bits<6> imm6;
3892
3893  let Inst{28} = U;
3894  let Inst{25-23} = 0b111;
3895  let Inst{22} = Qd{3};
3896  let Inst{21} = 0b1;
3897  let Inst{19-16} = imm6{3-0};
3898  let Inst{15-13} = Qd{2-0};
3899  let Inst{11-10} = 0b11;
3900  let Inst{9} = fsi;
3901  let Inst{8} = op;
3902  let Inst{7} = 0b0;
3903  let Inst{4} = 0b1;
3904
3905  let DecoderMethod = "DecodeMVEVCVTt1fp";
3906  let validForTailPredication = 1;
3907}
3908
3909class MVE_VCVT_imm_asmop<int Bits> : AsmOperandClass {
3910  let PredicateMethod = "isImmediate<1," # Bits # ">";
3911  let DiagnosticString =
3912      "MVE fixed-point immediate operand must be between 1 and " # Bits;
3913  let Name = "MVEVcvtImm" # Bits;
3914  let RenderMethod = "addImmOperands";
3915}
3916class MVE_VCVT_imm<int Bits>: Operand<i32> {
3917  let ParserMatchClass = MVE_VCVT_imm_asmop<Bits>;
3918  let EncoderMethod = "getNEONVcvtImm32OpValue";
3919  let DecoderMethod = "DecodeVCVTImmOperand";
3920}
3921
3922class MVE_VCVT_fix_f32<string suffix, bit U, bit op>
3923    : MVE_VCVT_fix<suffix, 0b1, U, op, MVE_VCVT_imm<32>> {
3924  let Inst{20} = imm6{4};
3925}
3926class MVE_VCVT_fix_f16<string suffix, bit U, bit op>
3927    : MVE_VCVT_fix<suffix, 0b0, U, op, MVE_VCVT_imm<16>> {
3928  let Inst{20} = 0b1;
3929}
3930
3931multiclass MVE_VCVT_fix_patterns<Instruction Inst, bit U, MVEVectorVTInfo DestVTI,
3932                                 MVEVectorVTInfo SrcVTI> {
3933  let Predicates = [HasMVEFloat] in {
3934    def : Pat<(DestVTI.Vec (int_arm_mve_vcvt_fix
3935                              (i32 U), (SrcVTI.Vec MQPR:$Qm), imm:$scale)),
3936              (DestVTI.Vec (Inst (SrcVTI.Vec MQPR:$Qm), imm:$scale))>;
3937    def : Pat<(DestVTI.Vec (int_arm_mve_vcvt_fix_predicated (i32 U),
3938                              (DestVTI.Vec MQPR:$inactive),
3939                              (SrcVTI.Vec MQPR:$Qm),
3940                              imm:$scale,
3941                              (DestVTI.Pred VCCR:$mask))),
3942              (DestVTI.Vec (Inst (SrcVTI.Vec MQPR:$Qm), imm:$scale,
3943                             ARMVCCThen, (DestVTI.Pred VCCR:$mask), zero_reg,
3944                             (DestVTI.Vec MQPR:$inactive)))>;
3945  }
3946}
3947
3948multiclass MVE_VCVT_fix_f32_m<bit U, bit op,
3949                              MVEVectorVTInfo DestVTI, MVEVectorVTInfo SrcVTI> {
3950  def "" : MVE_VCVT_fix_f32<DestVTI.Suffix#"."#SrcVTI.Suffix, U, op>;
3951  defm : MVE_VCVT_fix_patterns<!cast<Instruction>(NAME), U, DestVTI, SrcVTI>;
3952}
3953
3954multiclass MVE_VCVT_fix_f16_m<bit U, bit op,
3955                              MVEVectorVTInfo DestVTI, MVEVectorVTInfo SrcVTI> {
3956  def "" : MVE_VCVT_fix_f16<DestVTI.Suffix#"."#SrcVTI.Suffix, U, op>;
3957  defm : MVE_VCVT_fix_patterns<!cast<Instruction>(NAME), U, DestVTI, SrcVTI>;
3958}
3959
3960defm MVE_VCVTf16s16_fix : MVE_VCVT_fix_f16_m<0b0, 0b0, MVE_v8f16, MVE_v8s16>;
3961defm MVE_VCVTs16f16_fix : MVE_VCVT_fix_f16_m<0b0, 0b1, MVE_v8s16, MVE_v8f16>;
3962defm MVE_VCVTf16u16_fix : MVE_VCVT_fix_f16_m<0b1, 0b0, MVE_v8f16, MVE_v8u16>;
3963defm MVE_VCVTu16f16_fix : MVE_VCVT_fix_f16_m<0b1, 0b1, MVE_v8u16, MVE_v8f16>;
3964defm MVE_VCVTf32s32_fix : MVE_VCVT_fix_f32_m<0b0, 0b0, MVE_v4f32, MVE_v4s32>;
3965defm MVE_VCVTs32f32_fix : MVE_VCVT_fix_f32_m<0b0, 0b1, MVE_v4s32, MVE_v4f32>;
3966defm MVE_VCVTf32u32_fix : MVE_VCVT_fix_f32_m<0b1, 0b0, MVE_v4f32, MVE_v4u32>;
3967defm MVE_VCVTu32f32_fix : MVE_VCVT_fix_f32_m<0b1, 0b1, MVE_v4u32, MVE_v4f32>;
3968
3969class MVE_VCVT_fp_int_anpm<string suffix, bits<2> size, bit op, string anpm,
3970                bits<2> rm, list<dag> pattern=[]>
3971  : MVE_float<!strconcat("vcvt", anpm), suffix, (outs MQPR:$Qd),
3972              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {
3973  bits<4> Qd;
3974
3975  let Inst{28} = 0b1;
3976  let Inst{25-23} = 0b111;
3977  let Inst{22} = Qd{3};
3978  let Inst{21-20} = 0b11;
3979  let Inst{19-18} = size;
3980  let Inst{17-16} = 0b11;
3981  let Inst{15-13} = Qd{2-0};
3982  let Inst{12-10} = 0b000;
3983  let Inst{9-8} = rm;
3984  let Inst{7} = op;
3985  let Inst{4} = 0b0;
3986  let validForTailPredication = 1;
3987}
3988
3989multiclass MVE_VCVT_fp_int_anpm_inner<MVEVectorVTInfo Int, MVEVectorVTInfo Flt,
3990                                      string anpm, bits<2> rm> {
3991  def "": MVE_VCVT_fp_int_anpm<Int.Suffix # "." # Flt.Suffix, Int.Size,
3992                               Int.Unsigned, anpm, rm>;
3993
3994  defvar Inst         = !cast<Instruction>(NAME);
3995  defvar IntrBaseName = "int_arm_mve_vcvt" # anpm;
3996  defvar UnpredIntr   = !cast<Intrinsic>(IntrBaseName);
3997  defvar PredIntr     = !cast<Intrinsic>(IntrBaseName # "_predicated");
3998
3999  let Predicates = [HasMVEFloat] in {
4000    def : Pat<(Int.Vec (UnpredIntr (i32 Int.Unsigned), (Flt.Vec MQPR:$in))),
4001              (Int.Vec (Inst (Flt.Vec MQPR:$in)))>;
4002
4003    def : Pat<(Int.Vec (PredIntr (i32 Int.Unsigned), (Int.Vec MQPR:$inactive),
4004                                 (Flt.Vec MQPR:$in), (Flt.Pred VCCR:$pred))),
4005              (Int.Vec (Inst (Flt.Vec MQPR:$in), ARMVCCThen,
4006                             (Flt.Pred VCCR:$pred), zero_reg, (Int.Vec MQPR:$inactive)))>;
4007  }
4008}
4009
4010multiclass MVE_VCVT_fp_int_anpm_outer<MVEVectorVTInfo Int,
4011                                      MVEVectorVTInfo Flt> {
4012  defm a : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "a", 0b00>;
4013  defm n : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "n", 0b01>;
4014  defm p : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "p", 0b10>;
4015  defm m : MVE_VCVT_fp_int_anpm_inner<Int, Flt, "m", 0b11>;
4016}
4017
4018// This defines instructions such as MVE_VCVTu16f16a, with an explicit
4019// rounding-mode suffix on the mnemonic. The class below will define
4020// the bare MVE_VCVTu16f16 (with implied rounding toward zero).
4021defm MVE_VCVTs16f16 : MVE_VCVT_fp_int_anpm_outer<MVE_v8s16, MVE_v8f16>;
4022defm MVE_VCVTu16f16 : MVE_VCVT_fp_int_anpm_outer<MVE_v8u16, MVE_v8f16>;
4023defm MVE_VCVTs32f32 : MVE_VCVT_fp_int_anpm_outer<MVE_v4s32, MVE_v4f32>;
4024defm MVE_VCVTu32f32 : MVE_VCVT_fp_int_anpm_outer<MVE_v4u32, MVE_v4f32>;
4025
4026class MVE_VCVT_fp_int<string suffix, bits<2> size, bit toint, bit unsigned,
4027                      list<dag> pattern=[]>
4028  : MVE_float<"vcvt", suffix, (outs MQPR:$Qd),
4029              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {
4030  bits<4> Qd;
4031
4032  let Inst{28} = 0b1;
4033  let Inst{25-23} = 0b111;
4034  let Inst{22} = Qd{3};
4035  let Inst{21-20} = 0b11;
4036  let Inst{19-18} = size;
4037  let Inst{17-16} = 0b11;
4038  let Inst{15-13} = Qd{2-0};
4039  let Inst{12-9} = 0b0011;
4040  let Inst{8} = toint;
4041  let Inst{7} = unsigned;
4042  let Inst{4} = 0b0;
4043  let validForTailPredication = 1;
4044}
4045
4046multiclass MVE_VCVT_fp_int_m<MVEVectorVTInfo Dest, MVEVectorVTInfo Src,
4047                             SDNode unpred_op> {
4048  defvar Unsigned = !or(!eq(Dest.SuffixLetter,"u"), !eq(Src.SuffixLetter,"u"));
4049  defvar ToInt = !eq(Src.SuffixLetter,"f");
4050
4051  def "" : MVE_VCVT_fp_int<Dest.Suffix # "." # Src.Suffix, Dest.Size,
4052                           ToInt, Unsigned>;
4053  defvar Inst = !cast<Instruction>(NAME);
4054
4055  let Predicates = [HasMVEFloat] in {
4056    def : Pat<(Dest.Vec (unpred_op (Src.Vec MQPR:$src))),
4057              (Dest.Vec (Inst (Src.Vec MQPR:$src)))>;
4058    def : Pat<(Dest.Vec (int_arm_mve_vcvt_fp_int_predicated
4059                             (Src.Vec MQPR:$src), (i32 Unsigned),
4060                             (Src.Pred VCCR:$mask), (Dest.Vec MQPR:$inactive))),
4061              (Dest.Vec (Inst (Src.Vec MQPR:$src), ARMVCCThen,
4062                              (Src.Pred VCCR:$mask), zero_reg,
4063                              (Dest.Vec MQPR:$inactive)))>;
4064  }
4065}
4066// The unsuffixed VCVT for float->int implicitly rounds toward zero,
4067// which I reflect here in the llvm instruction names
4068defm MVE_VCVTs16f16z : MVE_VCVT_fp_int_m<MVE_v8s16, MVE_v8f16, fp_to_sint>;
4069defm MVE_VCVTu16f16z : MVE_VCVT_fp_int_m<MVE_v8u16, MVE_v8f16, fp_to_uint>;
4070defm MVE_VCVTs32f32z : MVE_VCVT_fp_int_m<MVE_v4s32, MVE_v4f32, fp_to_sint>;
4071defm MVE_VCVTu32f32z : MVE_VCVT_fp_int_m<MVE_v4u32, MVE_v4f32, fp_to_uint>;
4072// Whereas VCVT for int->float rounds to nearest
4073defm MVE_VCVTf16s16n : MVE_VCVT_fp_int_m<MVE_v8f16, MVE_v8s16, sint_to_fp>;
4074defm MVE_VCVTf16u16n : MVE_VCVT_fp_int_m<MVE_v8f16, MVE_v8u16, uint_to_fp>;
4075defm MVE_VCVTf32s32n : MVE_VCVT_fp_int_m<MVE_v4f32, MVE_v4s32, sint_to_fp>;
4076defm MVE_VCVTf32u32n : MVE_VCVT_fp_int_m<MVE_v4f32, MVE_v4u32, uint_to_fp>;
4077
4078let Predicates = [HasMVEFloat] in {
4079  def : Pat<(v4i32 (fp_to_sint_sat v4f32:$src, i32)),
4080            (MVE_VCVTs32f32z v4f32:$src)>;
4081  def : Pat<(v4i32 (fp_to_uint_sat v4f32:$src, i32)),
4082            (MVE_VCVTu32f32z v4f32:$src)>;
4083  def : Pat<(v8i16 (fp_to_sint_sat v8f16:$src, i16)),
4084            (MVE_VCVTs16f16z v8f16:$src)>;
4085  def : Pat<(v8i16 (fp_to_uint_sat v8f16:$src, i16)),
4086            (MVE_VCVTu16f16z v8f16:$src)>;
4087}
4088
4089class MVE_VABSNEG_fp<string iname, string suffix, bits<2> size, bit negate,
4090                   list<dag> pattern=[]>
4091  : MVE_float<iname, suffix, (outs MQPR:$Qd),
4092              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", size, pattern> {
4093  bits<4> Qd;
4094
4095  let Inst{28} = 0b1;
4096  let Inst{25-23} = 0b111;
4097  let Inst{22} = Qd{3};
4098  let Inst{21-20} = 0b11;
4099  let Inst{19-18} = size;
4100  let Inst{17-16} = 0b01;
4101  let Inst{15-13} = Qd{2-0};
4102  let Inst{11-8} = 0b0111;
4103  let Inst{7} = negate;
4104  let Inst{4} = 0b0;
4105  let validForTailPredication = 1;
4106}
4107
4108multiclass MVE_VABSNEG_fp_m<string iname, SDNode unpred_op, Intrinsic pred_int,
4109                            MVEVectorVTInfo VTI, bit opcode> {
4110  def "" : MVE_VABSNEG_fp<iname, VTI.Suffix, VTI.Size, opcode>;
4111  defvar Inst = !cast<Instruction>(NAME);
4112
4113  let Predicates = [HasMVEInt] in {
4114    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),
4115              (VTI.Vec (Inst $v))>;
4116    def : Pat<(VTI.Vec (pred_int  (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
4117                                  (VTI.Vec MQPR:$inactive))),
4118              (VTI.Vec (Inst $v, ARMVCCThen, $mask, zero_reg, $inactive))>;
4119  }
4120}
4121
4122defm MVE_VABSf16 : MVE_VABSNEG_fp_m<"vabs", fabs, int_arm_mve_abs_predicated,
4123                                    MVE_v8f16, 0>;
4124defm MVE_VABSf32 : MVE_VABSNEG_fp_m<"vabs", fabs, int_arm_mve_abs_predicated,
4125                                    MVE_v4f32, 0>;
4126defm MVE_VNEGf16 : MVE_VABSNEG_fp_m<"vneg", fneg, int_arm_mve_neg_predicated,
4127                                    MVE_v8f16, 1>;
4128defm MVE_VNEGf32 : MVE_VABSNEG_fp_m<"vneg", fneg, int_arm_mve_neg_predicated,
4129                                    MVE_v4f32, 1>;
4130
4131class MVE_VMAXMINNMA<string iname, string suffix, bits<2> size, bit bit_12,
4132                     list<dag> pattern=[]>
4133  : MVE_f<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
4134          NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",
4135          size, pattern> {
4136  bits<4> Qd;
4137  bits<4> Qm;
4138
4139  let Inst{28} = size{0};
4140  let Inst{25-23} = 0b100;
4141  let Inst{22} = Qd{3};
4142  let Inst{21-16} = 0b111111;
4143  let Inst{15-13} = Qd{2-0};
4144  let Inst{12} = bit_12;
4145  let Inst{11-6} = 0b111010;
4146  let Inst{5} = Qm{3};
4147  let Inst{4} = 0b0;
4148  let Inst{3-1} = Qm{2-0};
4149  let Inst{0} = 0b1;
4150
4151  let isCommutable = 1;
4152  let validForTailPredication = 1;
4153}
4154
4155multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,
4156                      SDNode unpred_op, Intrinsic pred_int,
4157                      bit bit_12> {
4158  def "" : MVE_VMAXMINNMA<iname, VTI.Suffix, VTI.Size, bit_12>;
4159  defvar Inst = !cast<Instruction>(NAME);
4160
4161  let Predicates = [HasMVEInt] in {
4162    // Unpredicated v(max|min)nma
4163    def : Pat<(VTI.Vec (unpred_op (fabs (VTI.Vec MQPR:$Qd)),
4164                                  (fabs (VTI.Vec MQPR:$Qm)))),
4165              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
4166
4167    // Predicated v(max|min)nma
4168    def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
4169                            (VTI.Pred VCCR:$mask))),
4170              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
4171                            ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
4172  }
4173}
4174
4175multiclass MVE_VMAXNMA<MVEVectorVTInfo VTI, bit bit_12>
4176  : MVE_VMAXMINNMA_m<"vmaxnma", VTI, fmaxnum, int_arm_mve_vmaxnma_predicated, bit_12>;
4177
4178defm MVE_VMAXNMAf32 : MVE_VMAXNMA<MVE_v4f32, 0b0>;
4179defm MVE_VMAXNMAf16 : MVE_VMAXNMA<MVE_v8f16, 0b0>;
4180
4181multiclass MVE_VMINNMA<MVEVectorVTInfo VTI, bit bit_12>
4182  : MVE_VMAXMINNMA_m<"vminnma", VTI, fminnum, int_arm_mve_vminnma_predicated, bit_12>;
4183
4184defm MVE_VMINNMAf32 : MVE_VMINNMA<MVE_v4f32, 0b1>;
4185defm MVE_VMINNMAf16 : MVE_VMINNMA<MVE_v8f16, 0b1>;
4186
4187// end of MVE Floating Point instructions
4188
4189// start of MVE compares
4190
4191class MVE_VCMPqq<string suffix, bit bit_28, bits<2> bits_21_20,
4192                 VCMPPredicateOperand predtype, bits<2> vecsize, list<dag> pattern=[]>
4193  : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, MQPR:$Qm, predtype:$fc),
4194           NoItinerary, "vcmp", suffix, "$fc, $Qn, $Qm", vpred_n, "", vecsize, pattern> {
4195  // Base class for comparing two vector registers
4196  bits<3> fc;
4197  bits<4> Qn;
4198  bits<4> Qm;
4199
4200  let Inst{28} = bit_28;
4201  let Inst{25-22} = 0b1000;
4202  let Inst{21-20} = bits_21_20;
4203  let Inst{19-17} = Qn{2-0};
4204  let Inst{16-13} = 0b1000;
4205  let Inst{12} = fc{2};
4206  let Inst{11-8} = 0b1111;
4207  let Inst{7} = fc{0};
4208  let Inst{6} = 0b0;
4209  let Inst{5} = Qm{3};
4210  let Inst{4} = 0b0;
4211  let Inst{3-1} = Qm{2-0};
4212  let Inst{0} = fc{1};
4213
4214  let Constraints = "";
4215
4216  // We need a custom decoder method for these instructions because of
4217  // the output VCCR operand, which isn't encoded in the instruction
4218  // bits anywhere (there is only one choice for it) but has to be
4219  // included in the MC operands so that codegen will be able to track
4220  // its data flow between instructions, spill/reload it when
4221  // necessary, etc. There seems to be no way to get the Tablegen
4222  // decoder to emit an operand that isn't affected by any instruction
4223  // bit.
4224  let DecoderMethod = "DecodeMVEVCMP<false," # predtype.DecoderMethod # ">";
4225  let validForTailPredication = 1;
4226}
4227
4228class MVE_VCMPqqf<string suffix, bit size>
4229    : MVE_VCMPqq<suffix, size, 0b11, pred_basic_fp, !if(size, 0b01, 0b10)> {
4230  let Predicates = [HasMVEFloat];
4231}
4232
4233class MVE_VCMPqqi<string suffix, bits<2> size>
4234    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_i, size> {
4235  let Inst{12} = 0b0;
4236  let Inst{0} = 0b0;
4237}
4238
4239class MVE_VCMPqqu<string suffix, bits<2> size>
4240    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_u, size> {
4241  let Inst{12} = 0b0;
4242  let Inst{0} = 0b1;
4243}
4244
4245class MVE_VCMPqqs<string suffix, bits<2> size>
4246    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_s, size> {
4247  let Inst{12} = 0b1;
4248}
4249
4250def MVE_VCMPf32 : MVE_VCMPqqf<"f32", 0b0>;
4251def MVE_VCMPf16 : MVE_VCMPqqf<"f16", 0b1>;
4252
4253def MVE_VCMPi8  : MVE_VCMPqqi<"i8",  0b00>;
4254def MVE_VCMPi16 : MVE_VCMPqqi<"i16", 0b01>;
4255def MVE_VCMPi32 : MVE_VCMPqqi<"i32", 0b10>;
4256
4257def MVE_VCMPu8  : MVE_VCMPqqu<"u8",  0b00>;
4258def MVE_VCMPu16 : MVE_VCMPqqu<"u16", 0b01>;
4259def MVE_VCMPu32 : MVE_VCMPqqu<"u32", 0b10>;
4260
4261def MVE_VCMPs8  : MVE_VCMPqqs<"s8",  0b00>;
4262def MVE_VCMPs16 : MVE_VCMPqqs<"s16", 0b01>;
4263def MVE_VCMPs32 : MVE_VCMPqqs<"s32", 0b10>;
4264
4265class MVE_VCMPqr<string suffix, bit bit_28, bits<2> bits_21_20,
4266                 VCMPPredicateOperand predtype, bits<2> vecsize, list<dag> pattern=[]>
4267  : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, GPRwithZR:$Rm, predtype:$fc),
4268           NoItinerary, "vcmp", suffix, "$fc, $Qn, $Rm", vpred_n, "", vecsize, pattern> {
4269  // Base class for comparing a vector register with a scalar
4270  bits<3> fc;
4271  bits<4> Qn;
4272  bits<4> Rm;
4273
4274  let Inst{28} = bit_28;
4275  let Inst{25-22} = 0b1000;
4276  let Inst{21-20} = bits_21_20;
4277  let Inst{19-17} = Qn{2-0};
4278  let Inst{16-13} = 0b1000;
4279  let Inst{12} = fc{2};
4280  let Inst{11-8} = 0b1111;
4281  let Inst{7} = fc{0};
4282  let Inst{6} = 0b1;
4283  let Inst{5} = fc{1};
4284  let Inst{4} = 0b0;
4285  let Inst{3-0} = Rm{3-0};
4286
4287  let Constraints = "";
4288  // Custom decoder method, for the same reason as MVE_VCMPqq
4289  let DecoderMethod = "DecodeMVEVCMP<true," # predtype.DecoderMethod # ">";
4290  let validForTailPredication = 1;
4291}
4292
4293class MVE_VCMPqrf<string suffix, bit size>
4294    : MVE_VCMPqr<suffix, size, 0b11, pred_basic_fp, !if(size, 0b01, 0b10)> {
4295  let Predicates = [HasMVEFloat];
4296}
4297
4298class MVE_VCMPqri<string suffix, bits<2> size>
4299    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_i, size> {
4300  let Inst{12} = 0b0;
4301  let Inst{5} = 0b0;
4302}
4303
4304class MVE_VCMPqru<string suffix, bits<2> size>
4305    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_u, size> {
4306  let Inst{12} = 0b0;
4307  let Inst{5} = 0b1;
4308}
4309
4310class MVE_VCMPqrs<string suffix, bits<2> size>
4311    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_s, size> {
4312  let Inst{12} = 0b1;
4313}
4314
4315def MVE_VCMPf32r : MVE_VCMPqrf<"f32", 0b0>;
4316def MVE_VCMPf16r : MVE_VCMPqrf<"f16", 0b1>;
4317
4318def MVE_VCMPi8r  : MVE_VCMPqri<"i8",  0b00>;
4319def MVE_VCMPi16r : MVE_VCMPqri<"i16", 0b01>;
4320def MVE_VCMPi32r : MVE_VCMPqri<"i32", 0b10>;
4321
4322def MVE_VCMPu8r  : MVE_VCMPqru<"u8",  0b00>;
4323def MVE_VCMPu16r : MVE_VCMPqru<"u16", 0b01>;
4324def MVE_VCMPu32r : MVE_VCMPqru<"u32", 0b10>;
4325
4326def MVE_VCMPs8r  : MVE_VCMPqrs<"s8",  0b00>;
4327def MVE_VCMPs16r : MVE_VCMPqrs<"s16", 0b01>;
4328def MVE_VCMPs32r : MVE_VCMPqrs<"s32", 0b10>;
4329
4330multiclass unpred_vcmp_z<string suffix, PatLeaf fc> {
4331  def i8  : Pat<(v16i1 (ARMvcmpz (v16i8 MQPR:$v1), fc)),
4332                (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc))>;
4333  def i16 : Pat<(v8i1 (ARMvcmpz (v8i16 MQPR:$v1), fc)),
4334                (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc))>;
4335  def i32 : Pat<(v4i1 (ARMvcmpz (v4i32 MQPR:$v1), fc)),
4336                (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc))>;
4337
4338  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmpz (v16i8 MQPR:$v1), fc)))),
4339            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4340  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8i16 MQPR:$v1), fc)))),
4341            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4342  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4i32 MQPR:$v1), fc)))),
4343            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4344}
4345
4346multiclass unpred_vcmp_r<string suffix, PatLeaf fc> {
4347  def i8  : Pat<(v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc)),
4348                (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc))>;
4349  def i16 : Pat<(v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc)),
4350                (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc))>;
4351  def i32 : Pat<(v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc)),
4352                (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc))>;
4353
4354  def i8r  : Pat<(v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 (ARMvdup rGPR:$v2)), fc)),
4355                 (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4356  def i16r : Pat<(v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 (ARMvdup rGPR:$v2)), fc)),
4357                 (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4358  def i32r : Pat<(v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 (ARMvdup rGPR:$v2)), fc)),
4359                 (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4360
4361  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc)))),
4362            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8") (v16i8 MQPR:$v1), (v16i8 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4363  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc)))),
4364            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16") (v8i16 MQPR:$v1), (v8i16 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4365  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc)))),
4366            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32") (v4i32 MQPR:$v1), (v4i32 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4367
4368  def : Pat<(v16i1 (and (v16i1 VCCR:$p1), (v16i1 (ARMvcmp (v16i8 MQPR:$v1), (v16i8 (ARMvdup rGPR:$v2)), fc)))),
4369            (v16i1 (!cast<Instruction>("MVE_VCMP"#suffix#"8r") (v16i8 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4370  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8i16 MQPR:$v1), (v8i16 (ARMvdup rGPR:$v2)), fc)))),
4371            (v8i1 (!cast<Instruction>("MVE_VCMP"#suffix#"16r") (v8i16 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4372  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4i32 MQPR:$v1), (v4i32 (ARMvdup rGPR:$v2)), fc)))),
4373            (v4i1 (!cast<Instruction>("MVE_VCMP"#suffix#"32r") (v4i32 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4374}
4375
4376multiclass unpred_vcmpf_z<PatLeaf fc> {
4377  def f16 : Pat<(v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)),
4378                (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc))>;
4379  def f32 : Pat<(v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)),
4380                (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc))>;
4381
4382  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmpz (v8f16 MQPR:$v1), fc)))),
4383            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4384  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmpz (v4f32 MQPR:$v1), fc)))),
4385            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4386}
4387
4388multiclass unpred_vcmpf_r<PatLeaf fc> {
4389  def : Pat<(v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc)),
4390            (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc))>;
4391  def : Pat<(v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc)),
4392            (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc))>;
4393
4394  def : Pat<(v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 (ARMvdup rGPR:$v2)), fc)),
4395            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4396  def : Pat<(v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 (ARMvdup rGPR:$v2)), fc)),
4397            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), (i32 rGPR:$v2), fc))>;
4398
4399  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc)))),
4400            (v8i1 (MVE_VCMPf16 (v8f16 MQPR:$v1), (v8f16 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4401  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc)))),
4402            (v4i1 (MVE_VCMPf32 (v4f32 MQPR:$v1), (v4f32 MQPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4403
4404  def : Pat<(v8i1 (and (v8i1 VCCR:$p1), (v8i1 (ARMvcmp (v8f16 MQPR:$v1), (v8f16 (ARMvdup rGPR:$v2)), fc)))),
4405            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4406  def : Pat<(v4i1 (and (v4i1 VCCR:$p1), (v4i1 (ARMvcmp (v4f32 MQPR:$v1), (v4f32 (ARMvdup rGPR:$v2)), fc)))),
4407            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), (i32 rGPR:$v2), fc, ARMVCCThen, VCCR:$p1, zero_reg))>;
4408}
4409
4410let Predicates = [HasMVEInt] in {
4411  defm MVE_VCEQZ  : unpred_vcmp_z<"i", ARMCCeq>;
4412  defm MVE_VCNEZ  : unpred_vcmp_z<"i", ARMCCne>;
4413  defm MVE_VCGEZ  : unpred_vcmp_z<"s", ARMCCge>;
4414  defm MVE_VCLTZ  : unpred_vcmp_z<"s", ARMCClt>;
4415  defm MVE_VCGTZ  : unpred_vcmp_z<"s", ARMCCgt>;
4416  defm MVE_VCLEZ  : unpred_vcmp_z<"s", ARMCCle>;
4417  defm MVE_VCGTUZ : unpred_vcmp_z<"u", ARMCChi>;
4418  defm MVE_VCGEUZ : unpred_vcmp_z<"u", ARMCChs>;
4419
4420  defm MVE_VCEQ   : unpred_vcmp_r<"i", ARMCCeq>;
4421  defm MVE_VCNE   : unpred_vcmp_r<"i", ARMCCne>;
4422  defm MVE_VCGE   : unpred_vcmp_r<"s", ARMCCge>;
4423  defm MVE_VCLT   : unpred_vcmp_r<"s", ARMCClt>;
4424  defm MVE_VCGT   : unpred_vcmp_r<"s", ARMCCgt>;
4425  defm MVE_VCLE   : unpred_vcmp_r<"s", ARMCCle>;
4426  defm MVE_VCGTU  : unpred_vcmp_r<"u", ARMCChi>;
4427  defm MVE_VCGEU  : unpred_vcmp_r<"u", ARMCChs>;
4428}
4429
4430let Predicates = [HasMVEFloat] in {
4431  defm MVE_VFCEQZ  : unpred_vcmpf_z<ARMCCeq>;
4432  defm MVE_VFCNEZ  : unpred_vcmpf_z<ARMCCne>;
4433  defm MVE_VFCGEZ  : unpred_vcmpf_z<ARMCCge>;
4434  defm MVE_VFCLTZ  : unpred_vcmpf_z<ARMCClt>;
4435  defm MVE_VFCGTZ  : unpred_vcmpf_z<ARMCCgt>;
4436  defm MVE_VFCLEZ  : unpred_vcmpf_z<ARMCCle>;
4437
4438  defm MVE_VFCEQ   : unpred_vcmpf_r<ARMCCeq>;
4439  defm MVE_VFCNE   : unpred_vcmpf_r<ARMCCne>;
4440  defm MVE_VFCGE   : unpred_vcmpf_r<ARMCCge>;
4441  defm MVE_VFCLT   : unpred_vcmpf_r<ARMCClt>;
4442  defm MVE_VFCGT   : unpred_vcmpf_r<ARMCCgt>;
4443  defm MVE_VFCLE   : unpred_vcmpf_r<ARMCCle>;
4444}
4445
4446
4447// Extra "worst case" and/or/xor patterns, going into and out of GRP
4448multiclass two_predops<SDPatternOperator opnode, Instruction insn> {
4449  def v16i1 : Pat<(v16i1 (opnode (v16i1 VCCR:$p1), (v16i1 VCCR:$p2))),
4450                  (v16i1 (COPY_TO_REGCLASS
4451                           (insn (i32 (COPY_TO_REGCLASS (v16i1 VCCR:$p1), rGPR)),
4452                                 (i32 (COPY_TO_REGCLASS (v16i1 VCCR:$p2), rGPR))),
4453                           VCCR))>;
4454  def v8i1  : Pat<(v8i1 (opnode (v8i1 VCCR:$p1), (v8i1 VCCR:$p2))),
4455                  (v8i1 (COPY_TO_REGCLASS
4456                          (insn (i32 (COPY_TO_REGCLASS (v8i1 VCCR:$p1), rGPR)),
4457                                (i32 (COPY_TO_REGCLASS (v8i1 VCCR:$p2), rGPR))),
4458                          VCCR))>;
4459  def v4i1  : Pat<(v4i1 (opnode (v4i1 VCCR:$p1), (v4i1 VCCR:$p2))),
4460                  (v4i1 (COPY_TO_REGCLASS
4461                          (insn (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p1), rGPR)),
4462                                (i32 (COPY_TO_REGCLASS (v4i1 VCCR:$p2), rGPR))),
4463                          VCCR))>;
4464  def v2i1  : Pat<(v2i1 (opnode (v2i1 VCCR:$p1), (v2i1 VCCR:$p2))),
4465                  (v2i1 (COPY_TO_REGCLASS
4466                          (insn (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p1), rGPR)),
4467                                (i32 (COPY_TO_REGCLASS (v2i1 VCCR:$p2), rGPR))),
4468                          VCCR))>;
4469}
4470
4471let Predicates = [HasMVEInt] in {
4472  defm POR    : two_predops<or,  t2ORRrr>;
4473  defm PAND   : two_predops<and, t2ANDrr>;
4474  defm PEOR   : two_predops<xor, t2EORrr>;
4475}
4476
4477// Occasionally we need to cast between a i32 and a boolean vector, for
4478// example when moving between rGPR and VPR.P0 as part of predicate vector
4479// shuffles. We also sometimes need to cast between different predicate
4480// vector types (v4i1<>v8i1, etc.) also as part of lowering vector shuffles.
4481def predicate_cast : SDNode<"ARMISD::PREDICATE_CAST", SDTUnaryOp>;
4482
4483def load_align4 : PatFrag<(ops node:$ptr), (load node:$ptr), [{
4484  return cast<LoadSDNode>(N)->getAlignment() >= 4;
4485}]>;
4486
4487let Predicates = [HasMVEInt] in {
4488  foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in {
4489    def : Pat<(i32 (predicate_cast (VT VCCR:$src))),
4490              (i32 (COPY_TO_REGCLASS (VT VCCR:$src), VCCR))>;
4491    def : Pat<(VT  (predicate_cast (i32 VCCR:$src))),
4492              (VT  (COPY_TO_REGCLASS (i32 VCCR:$src), VCCR))>;
4493
4494    foreach VT2 = [ v2i1, v4i1, v8i1, v16i1 ] in
4495      def : Pat<(VT  (predicate_cast (VT2 VCCR:$src))),
4496                (VT  (COPY_TO_REGCLASS (VT2 VCCR:$src), VCCR))>;
4497  }
4498
4499  // If we happen to be casting from a load we can convert that straight
4500  // into a predicate load, so long as the load is of the correct type.
4501  foreach VT = [ v2i1, v4i1, v8i1, v16i1 ] in {
4502    def : Pat<(VT (predicate_cast (i32 (load_align4 taddrmode_imm7<2>:$addr)))),
4503              (VT (VLDR_P0_off taddrmode_imm7<2>:$addr))>;
4504  }
4505
4506  // Here we match the specific SDNode type 'ARMVectorRegCastImpl'
4507  // rather than the more general 'ARMVectorRegCast' which would also
4508  // match some bitconverts. If we use the latter in cases where the
4509  // input and output types are the same, the bitconvert gets elided
4510  // and we end up generating a nonsense match of nothing.
4511
4512  foreach VT = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in
4513    foreach VT2 = [ v16i8, v8i16, v8f16, v4i32, v4f32, v2i64, v2f64 ] in
4514      def : Pat<(VT (ARMVectorRegCastImpl (VT2 MQPR:$src))),
4515                (VT MQPR:$src)>;
4516}
4517
4518// end of MVE compares
4519
4520// start of MVE_qDest_qSrc
4521
4522class MVE_qDest_qSrc<string iname, string suffix, dag oops, dag iops,
4523                     string ops, vpred_ops vpred, string cstr,
4524                     bits<2> vecsize, list<dag> pattern=[]>
4525  : MVE_p<oops, iops, NoItinerary, iname, suffix,
4526          ops, vpred, cstr, vecsize, pattern> {
4527  bits<4> Qd;
4528  bits<4> Qm;
4529
4530  let Inst{25-23} = 0b100;
4531  let Inst{22} = Qd{3};
4532  let Inst{15-13} = Qd{2-0};
4533  let Inst{11-9} = 0b111;
4534  let Inst{6} = 0b0;
4535  let Inst{5} = Qm{3};
4536  let Inst{4} = 0b0;
4537  let Inst{3-1} = Qm{2-0};
4538}
4539
4540class MVE_VQxDMLxDH<string iname, bit exch, bit round, bit subtract,
4541                    string suffix, bits<2> size, string cstr="",
4542                    list<dag> pattern=[]>
4543  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4544                   (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4545                   vpred_n, "$Qd = $Qd_src"#cstr, size, pattern> {
4546  bits<4> Qn;
4547
4548  let Inst{28} = subtract;
4549  let Inst{21-20} = size;
4550  let Inst{19-17} = Qn{2-0};
4551  let Inst{16} = 0b0;
4552  let Inst{12} = exch;
4553  let Inst{8} = 0b0;
4554  let Inst{7} = Qn{3};
4555  let Inst{0} = round;
4556}
4557
4558multiclass MVE_VQxDMLxDH_p<string iname, bit exch, bit round, bit subtract,
4559                           MVEVectorVTInfo VTI> {
4560  def "": MVE_VQxDMLxDH<iname, exch, round, subtract, VTI.Suffix, VTI.Size,
4561                        !if(!eq(VTI.LaneBits, 32), ",@earlyclobber $Qd", "")>;
4562  defvar Inst = !cast<Instruction>(NAME);
4563  defvar ConstParams = (? (i32 exch), (i32 round), (i32 subtract));
4564  defvar unpred_intr = int_arm_mve_vqdmlad;
4565  defvar pred_intr = int_arm_mve_vqdmlad_predicated;
4566
4567  def : Pat<(VTI.Vec !con((unpred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4568                                       (VTI.Vec MQPR:$c)), ConstParams)),
4569            (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4570                           (VTI.Vec MQPR:$c)))>;
4571  def : Pat<(VTI.Vec !con((pred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4572                                     (VTI.Vec MQPR:$c)), ConstParams,
4573                          (? (VTI.Pred VCCR:$pred)))),
4574            (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4575                           (VTI.Vec MQPR:$c),
4576                           ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
4577}
4578
4579multiclass MVE_VQxDMLxDH_multi<string iname, bit exch,
4580                               bit round, bit subtract> {
4581  defm s8  : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v16s8>;
4582  defm s16 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v8s16>;
4583  defm s32 : MVE_VQxDMLxDH_p<iname, exch, round, subtract, MVE_v4s32>;
4584}
4585
4586defm MVE_VQDMLADH   : MVE_VQxDMLxDH_multi<"vqdmladh",   0b0, 0b0, 0b0>;
4587defm MVE_VQDMLADHX  : MVE_VQxDMLxDH_multi<"vqdmladhx",  0b1, 0b0, 0b0>;
4588defm MVE_VQRDMLADH  : MVE_VQxDMLxDH_multi<"vqrdmladh",  0b0, 0b1, 0b0>;
4589defm MVE_VQRDMLADHX : MVE_VQxDMLxDH_multi<"vqrdmladhx", 0b1, 0b1, 0b0>;
4590defm MVE_VQDMLSDH   : MVE_VQxDMLxDH_multi<"vqdmlsdh",   0b0, 0b0, 0b1>;
4591defm MVE_VQDMLSDHX  : MVE_VQxDMLxDH_multi<"vqdmlsdhx",  0b1, 0b0, 0b1>;
4592defm MVE_VQRDMLSDH  : MVE_VQxDMLxDH_multi<"vqrdmlsdh",  0b0, 0b1, 0b1>;
4593defm MVE_VQRDMLSDHX : MVE_VQxDMLxDH_multi<"vqrdmlsdhx", 0b1, 0b1, 0b1>;
4594
4595class MVE_VCMUL<string iname, string suffix, bits<2> size, string cstr="">
4596  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4597                   (ins MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
4598                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, size,
4599                   []> {
4600  bits<4> Qn;
4601  bits<2> rot;
4602
4603  let Inst{28} = size{1};
4604  let Inst{21-20} = 0b11;
4605  let Inst{19-17} = Qn{2-0};
4606  let Inst{16} = 0b0;
4607  let Inst{12} = rot{1};
4608  let Inst{8} = 0b0;
4609  let Inst{7} = Qn{3};
4610  let Inst{0} = rot{0};
4611
4612  let Predicates = [HasMVEFloat];
4613}
4614
4615multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,
4616                       string cstr=""> {
4617  def "" : MVE_VCMUL<iname, VTI.Suffix, VTI.Size, cstr>;
4618  defvar Inst = !cast<Instruction>(NAME);
4619
4620  let Predicates = [HasMVEFloat] in {
4621    def : Pat<(VTI.Vec (int_arm_mve_vcmulq
4622                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
4623              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4624                             imm:$rot))>;
4625
4626    def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated
4627                            imm:$rot, (VTI.Vec MQPR:$inactive),
4628                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4629                            (VTI.Pred VCCR:$mask))),
4630              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4631                             imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
4632                             (VTI.Vec MQPR:$inactive)))>;
4633
4634  }
4635}
4636
4637defm MVE_VCMULf16 : MVE_VCMUL_m<"vcmul", MVE_v8f16>;
4638defm MVE_VCMULf32 : MVE_VCMUL_m<"vcmul", MVE_v4f32, "@earlyclobber $Qd">;
4639
4640class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,
4641                bit T, string cstr, bits<2> vecsize, list<dag> pattern=[]>
4642  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4643                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4644                   vpred_r, cstr, vecsize, pattern> {
4645  bits<4> Qd;
4646  bits<4> Qn;
4647  bits<4> Qm;
4648
4649  let Inst{28} = bit_28;
4650  let Inst{21-20} = bits_21_20;
4651  let Inst{19-17} = Qn{2-0};
4652  let Inst{16} = 0b1;
4653  let Inst{12} = T;
4654  let Inst{8} = 0b0;
4655  let Inst{7} = Qn{3};
4656  let Inst{0} = 0b0;
4657  let validForTailPredication = 1;
4658  let doubleWidthResult = 1;
4659}
4660
4661multiclass MVE_VMULL_m<MVEVectorVTInfo VTI,
4662                       SDPatternOperator unpred_op, Intrinsic pred_int,
4663                       bit Top, bits<2> vecsize, string cstr=""> {
4664  def "" : MVE_VMULL<"vmull" # !if(Top, "t", "b"), VTI.Suffix, VTI.Unsigned,
4665                     VTI.Size, Top, cstr, vecsize>;
4666  defvar Inst = !cast<Instruction>(NAME);
4667
4668  let Predicates = [HasMVEInt] in {
4669    defvar uflag = !if(!eq(VTI.SuffixLetter, "p"), (?), (? (i32 VTI.Unsigned)));
4670
4671    // Unpredicated multiply
4672    def : Pat<(VTI.DblVec !con((unpred_op (VTI.Vec MQPR:$Qm),
4673                                          (VTI.Vec MQPR:$Qn)),
4674                               uflag, (? (i32 Top)))),
4675              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4676
4677    // Predicated multiply
4678    def : Pat<(VTI.DblVec !con((pred_int (VTI.Vec MQPR:$Qm),
4679                                         (VTI.Vec MQPR:$Qn)),
4680                               uflag, (? (i32 Top), (VTI.DblPred VCCR:$mask),
4681                                         (VTI.DblVec MQPR:$inactive)))),
4682              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4683                                ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
4684                                (VTI.DblVec MQPR:$inactive)))>;
4685  }
4686}
4687
4688// For polynomial multiplies, the size bits take the unused value 0b11, and
4689// the unsigned bit switches to encoding the size.
4690
4691defm MVE_VMULLBs8  : MVE_VMULL_m<MVE_v16s8, int_arm_mve_vmull,
4692                                 int_arm_mve_mull_int_predicated, 0b0, 0b01>;
4693defm MVE_VMULLTs8  : MVE_VMULL_m<MVE_v16s8, int_arm_mve_vmull,
4694                                 int_arm_mve_mull_int_predicated, 0b1, 0b01>;
4695defm MVE_VMULLBs16 : MVE_VMULL_m<MVE_v8s16, int_arm_mve_vmull,
4696                                 int_arm_mve_mull_int_predicated, 0b0, 0b10>;
4697defm MVE_VMULLTs16 : MVE_VMULL_m<MVE_v8s16, int_arm_mve_vmull,
4698                                 int_arm_mve_mull_int_predicated, 0b1, 0b10>;
4699defm MVE_VMULLBs32 : MVE_VMULL_m<MVE_v4s32, int_arm_mve_vmull,
4700                                 int_arm_mve_mull_int_predicated, 0b0, 0b11,
4701                                 "@earlyclobber $Qd">;
4702defm MVE_VMULLTs32 : MVE_VMULL_m<MVE_v4s32, int_arm_mve_vmull,
4703                                 int_arm_mve_mull_int_predicated, 0b1, 0b11,
4704                                 "@earlyclobber $Qd">;
4705
4706defm MVE_VMULLBu8  : MVE_VMULL_m<MVE_v16u8, int_arm_mve_vmull,
4707                                 int_arm_mve_mull_int_predicated, 0b0, 0b01>;
4708defm MVE_VMULLTu8  : MVE_VMULL_m<MVE_v16u8, int_arm_mve_vmull,
4709                                 int_arm_mve_mull_int_predicated, 0b1, 0b01>;
4710defm MVE_VMULLBu16 : MVE_VMULL_m<MVE_v8u16, int_arm_mve_vmull,
4711                                 int_arm_mve_mull_int_predicated, 0b0, 0b10>;
4712defm MVE_VMULLTu16 : MVE_VMULL_m<MVE_v8u16, int_arm_mve_vmull,
4713                                 int_arm_mve_mull_int_predicated, 0b1, 0b10>;
4714defm MVE_VMULLBu32 : MVE_VMULL_m<MVE_v4u32, int_arm_mve_vmull,
4715                                 int_arm_mve_mull_int_predicated, 0b0, 0b11,
4716                                 "@earlyclobber $Qd">;
4717defm MVE_VMULLTu32 : MVE_VMULL_m<MVE_v4u32, int_arm_mve_vmull,
4718                                 int_arm_mve_mull_int_predicated, 0b1, 0b11,
4719                                 "@earlyclobber $Qd">;
4720
4721defm MVE_VMULLBp8  : MVE_VMULL_m<MVE_v16p8, int_arm_mve_vmull_poly,
4722                                 int_arm_mve_mull_poly_predicated, 0b0, 0b01>;
4723defm MVE_VMULLTp8  : MVE_VMULL_m<MVE_v16p8, int_arm_mve_vmull_poly,
4724                                 int_arm_mve_mull_poly_predicated, 0b1, 0b01>;
4725defm MVE_VMULLBp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,
4726                                 int_arm_mve_mull_poly_predicated, 0b0, 0b10>;
4727defm MVE_VMULLTp16 : MVE_VMULL_m<MVE_v8p16, int_arm_mve_vmull_poly,
4728                                 int_arm_mve_mull_poly_predicated, 0b1, 0b10>;
4729
4730let Predicates = [HasMVEInt] in {
4731  def : Pat<(v2i64 (ARMvmulls (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),
4732            (MVE_VMULLBs32 MQPR:$src1, MQPR:$src2)>;
4733  def : Pat<(v2i64 (ARMvmulls (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),
4734                              (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),
4735            (MVE_VMULLTs32 MQPR:$src1, MQPR:$src2)>;
4736
4737  def : Pat<(mul (sext_inreg (v4i32 MQPR:$src1), v4i16),
4738                 (sext_inreg (v4i32 MQPR:$src2), v4i16)),
4739            (MVE_VMULLBs16 MQPR:$src1, MQPR:$src2)>;
4740  def : Pat<(mul (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))), v4i16),
4741                 (sext_inreg (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))), v4i16)),
4742            (MVE_VMULLTs16 MQPR:$src1, MQPR:$src2)>;
4743
4744  def : Pat<(mul (sext_inreg (v8i16 MQPR:$src1), v8i8),
4745                 (sext_inreg (v8i16 MQPR:$src2), v8i8)),
4746            (MVE_VMULLBs8 MQPR:$src1, MQPR:$src2)>;
4747  def : Pat<(mul (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), v8i8),
4748                 (sext_inreg (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), v8i8)),
4749            (MVE_VMULLTs8 MQPR:$src1, MQPR:$src2)>;
4750
4751  def : Pat<(v2i64 (ARMvmullu (v4i32 MQPR:$src1), (v4i32 MQPR:$src2))),
4752            (MVE_VMULLBu32 MQPR:$src1, MQPR:$src2)>;
4753  def : Pat<(v2i64 (ARMvmullu (v4i32 (ARMvrev64 (v4i32 MQPR:$src1))),
4754                              (v4i32 (ARMvrev64 (v4i32 MQPR:$src2))))),
4755            (MVE_VMULLTu32 MQPR:$src1, MQPR:$src2)>;
4756
4757  def : Pat<(mul (and (v4i32 MQPR:$src1), (v4i32 (ARMvmovImm (i32 0xCFF)))),
4758                 (and (v4i32 MQPR:$src2), (v4i32 (ARMvmovImm (i32 0xCFF))))),
4759            (MVE_VMULLBu16 MQPR:$src1, MQPR:$src2)>;
4760  def : Pat<(mul (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src1)))),
4761                      (v4i32 (ARMvmovImm (i32 0xCFF)))),
4762                 (and (v4i32 (ARMVectorRegCast (ARMvrev32 (v8i16 MQPR:$src2)))),
4763                      (v4i32 (ARMvmovImm (i32 0xCFF))))),
4764            (MVE_VMULLTu16 MQPR:$src1, MQPR:$src2)>;
4765
4766  def : Pat<(mul (ARMvbicImm (v8i16 MQPR:$src1), (i32 0xAFF)),
4767                 (ARMvbicImm (v8i16 MQPR:$src2), (i32 0xAFF))),
4768            (MVE_VMULLBu8 MQPR:$src1, MQPR:$src2)>;
4769  def : Pat<(mul (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src1)))), (i32 0xAFF)),
4770                 (ARMvbicImm (v8i16 (ARMVectorRegCast (ARMvrev16 (v16i8 MQPR:$src2)))), (i32 0xAFF))),
4771            (MVE_VMULLTu8 MQPR:$src1, MQPR:$src2)>;
4772}
4773
4774class MVE_VxMULH<string iname, string suffix, bit U, bits<2> size, bit round,
4775                 list<dag> pattern=[]>
4776  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4777                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
4778                   vpred_r, "", size, pattern> {
4779  bits<4> Qn;
4780
4781  let Inst{28} = U;
4782  let Inst{21-20} = size;
4783  let Inst{19-17} = Qn{2-0};
4784  let Inst{16} = 0b1;
4785  let Inst{12} = round;
4786  let Inst{8} = 0b0;
4787  let Inst{7} = Qn{3};
4788  let Inst{0} = 0b1;
4789  let validForTailPredication = 1;
4790}
4791
4792multiclass MVE_VxMULH_m<string iname, MVEVectorVTInfo VTI, SDNode unpred_op,
4793                        Intrinsic PredInt, bit round> {
4794  def "" : MVE_VxMULH<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, round>;
4795  defvar Inst = !cast<Instruction>(NAME);
4796
4797  let Predicates = [HasMVEInt] in {
4798    if !eq(round, 0b0) then {
4799      defvar mulh = !if(VTI.Unsigned, mulhu, mulhs);
4800      defm : MVE_TwoOpPattern<VTI, mulh, PredInt, (? (i32 VTI.Unsigned)),
4801                              !cast<Instruction>(NAME)>;
4802    } else {
4803      // Predicated multiply returning high bits
4804      def : Pat<(VTI.Vec (PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4805                              (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
4806                              (VTI.Vec MQPR:$inactive))),
4807                (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4808                              ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
4809                              (VTI.Vec MQPR:$inactive)))>;
4810    }
4811
4812    // Unpredicated intrinsic
4813    def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4814                            (i32 VTI.Unsigned))),
4815              (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4816  }
4817}
4818
4819multiclass MVE_VMULT<string iname, MVEVectorVTInfo VTI, bit round>
4820  : MVE_VxMULH_m<iname, VTI, !if(round, int_arm_mve_vrmulh, int_arm_mve_vmulh),
4821                 !if(round, int_arm_mve_rmulh_predicated,
4822                            int_arm_mve_mulh_predicated),
4823                 round>;
4824
4825defm MVE_VMULHs8   : MVE_VMULT<"vmulh",  MVE_v16s8, 0b0>;
4826defm MVE_VMULHs16  : MVE_VMULT<"vmulh",  MVE_v8s16, 0b0>;
4827defm MVE_VMULHs32  : MVE_VMULT<"vmulh",  MVE_v4s32, 0b0>;
4828defm MVE_VMULHu8   : MVE_VMULT<"vmulh",  MVE_v16u8, 0b0>;
4829defm MVE_VMULHu16  : MVE_VMULT<"vmulh",  MVE_v8u16, 0b0>;
4830defm MVE_VMULHu32  : MVE_VMULT<"vmulh",  MVE_v4u32, 0b0>;
4831
4832defm MVE_VRMULHs8  : MVE_VMULT<"vrmulh", MVE_v16s8, 0b1>;
4833defm MVE_VRMULHs16 : MVE_VMULT<"vrmulh", MVE_v8s16, 0b1>;
4834defm MVE_VRMULHs32 : MVE_VMULT<"vrmulh", MVE_v4s32, 0b1>;
4835defm MVE_VRMULHu8  : MVE_VMULT<"vrmulh", MVE_v16u8, 0b1>;
4836defm MVE_VRMULHu16 : MVE_VMULT<"vrmulh", MVE_v8u16, 0b1>;
4837defm MVE_VRMULHu32 : MVE_VMULT<"vrmulh", MVE_v4u32, 0b1>;
4838
4839class MVE_VxMOVxN<string iname, string suffix, bit bit_28, bit bit_17,
4840                  bits<2> size, bit T, list<dag> pattern=[]>
4841  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4842                   (ins MQPR:$Qd_src, MQPR:$Qm), "$Qd, $Qm",
4843                   vpred_n, "$Qd = $Qd_src", !if(size, 0b10, 0b01), pattern> {
4844
4845  let Inst{28} = bit_28;
4846  let Inst{21-20} = 0b11;
4847  let Inst{19-18} = size;
4848  let Inst{17} = bit_17;
4849  let Inst{16} = 0b1;
4850  let Inst{12} = T;
4851  let Inst{8} = 0b0;
4852  let Inst{7} = !not(bit_17);
4853  let Inst{0} = 0b1;
4854  let validForTailPredication = 1;
4855  let retainsPreviousHalfElement = 1;
4856}
4857
4858multiclass MVE_VxMOVxN_halves<string iname, string suffix,
4859                              bit bit_28, bit bit_17, bits<2> size> {
4860  def bh : MVE_VxMOVxN<iname # "b", suffix, bit_28, bit_17, size, 0b0>;
4861  def th : MVE_VxMOVxN<iname # "t", suffix, bit_28, bit_17, size, 0b1>;
4862}
4863
4864defm MVE_VMOVNi16   : MVE_VxMOVxN_halves<"vmovn",   "i16", 0b1, 0b0, 0b00>;
4865defm MVE_VMOVNi32   : MVE_VxMOVxN_halves<"vmovn",   "i32", 0b1, 0b0, 0b01>;
4866defm MVE_VQMOVNs16  : MVE_VxMOVxN_halves<"vqmovn",  "s16", 0b0, 0b1, 0b00>;
4867defm MVE_VQMOVNs32  : MVE_VxMOVxN_halves<"vqmovn",  "s32", 0b0, 0b1, 0b01>;
4868defm MVE_VQMOVNu16  : MVE_VxMOVxN_halves<"vqmovn",  "u16", 0b1, 0b1, 0b00>;
4869defm MVE_VQMOVNu32  : MVE_VxMOVxN_halves<"vqmovn",  "u32", 0b1, 0b1, 0b01>;
4870defm MVE_VQMOVUNs16 : MVE_VxMOVxN_halves<"vqmovun", "s16", 0b0, 0b0, 0b00>;
4871defm MVE_VQMOVUNs32 : MVE_VxMOVxN_halves<"vqmovun", "s32", 0b0, 0b0, 0b01>;
4872
4873def MVEvmovn       : SDNode<"ARMISD::VMOVN", SDTARMVEXT>;
4874
4875multiclass MVE_VMOVN_p<Instruction Inst, bit top,
4876                       MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4877  // Match the most obvious MVEvmovn(a,b,t), which overwrites the odd or even
4878  // lanes of a (depending on t) with the even lanes of b.
4879  def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qd_src),
4880                               (VTI.Vec MQPR:$Qm), (i32 top))),
4881            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4882
4883  if !not(top) then {
4884    // If we see MVEvmovn(a,ARMvrev(b),1), that wants to overwrite the odd
4885    // lanes of a with the odd lanes of b. In other words, the lanes we're
4886    // _keeping_ from a are the even ones. So we can flip it round and say that
4887    // this is the same as overwriting the even lanes of b with the even lanes
4888    // of a, i.e. it's a VMOVNB with the operands reversed.
4889    defvar vrev = !cast<SDNode>("ARMvrev" # InVTI.LaneBits);
4890    def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qm),
4891                                 (VTI.Vec (vrev MQPR:$Qd_src)), (i32 1))),
4892              (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4893  }
4894
4895  // Match the IR intrinsic for a predicated VMOVN. This regards the Qm input
4896  // as having wider lanes that we're narrowing, instead of already-narrow
4897  // lanes that we're taking every other one of.
4898  def : Pat<(VTI.Vec (int_arm_mve_vmovn_predicated (VTI.Vec MQPR:$Qd_src),
4899                                  (InVTI.Vec MQPR:$Qm), (i32 top),
4900                                  (InVTI.Pred VCCR:$pred))),
4901            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4902                              (InVTI.Vec MQPR:$Qm),
4903                              ARMVCCThen, (InVTI.Pred VCCR:$pred), zero_reg))>;
4904}
4905
4906defm : MVE_VMOVN_p<MVE_VMOVNi32bh, 0, MVE_v8i16, MVE_v4i32>;
4907defm : MVE_VMOVN_p<MVE_VMOVNi32th, 1, MVE_v8i16, MVE_v4i32>;
4908defm : MVE_VMOVN_p<MVE_VMOVNi16bh, 0, MVE_v16i8, MVE_v8i16>;
4909defm : MVE_VMOVN_p<MVE_VMOVNi16th, 1, MVE_v16i8, MVE_v8i16>;
4910
4911multiclass MVE_VQMOVN_p<Instruction Inst, bit outU, bit inU, bit top,
4912                        MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4913  def : Pat<(VTI.Vec (int_arm_mve_vqmovn (VTI.Vec MQPR:$Qd_src),
4914                                  (InVTI.Vec MQPR:$Qm),
4915                                  (i32 outU), (i32 inU), (i32 top))),
4916            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4917                              (InVTI.Vec MQPR:$Qm)))>;
4918
4919  def : Pat<(VTI.Vec (int_arm_mve_vqmovn_predicated (VTI.Vec MQPR:$Qd_src),
4920                                  (InVTI.Vec MQPR:$Qm),
4921                                  (i32 outU), (i32 inU), (i32 top),
4922                                  (InVTI.Pred VCCR:$pred))),
4923            (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4924                              (InVTI.Vec MQPR:$Qm),
4925                              ARMVCCThen, (InVTI.Pred VCCR:$pred), zero_reg))>;
4926}
4927
4928defm : MVE_VQMOVN_p<MVE_VQMOVNs32bh,  0, 0, 0, MVE_v8i16, MVE_v4i32>;
4929defm : MVE_VQMOVN_p<MVE_VQMOVNs32th,  0, 0, 1, MVE_v8i16, MVE_v4i32>;
4930defm : MVE_VQMOVN_p<MVE_VQMOVNs16bh,  0, 0, 0, MVE_v16i8, MVE_v8i16>;
4931defm : MVE_VQMOVN_p<MVE_VQMOVNs16th,  0, 0, 1, MVE_v16i8, MVE_v8i16>;
4932defm : MVE_VQMOVN_p<MVE_VQMOVNu32bh,  1, 1, 0, MVE_v8i16, MVE_v4i32>;
4933defm : MVE_VQMOVN_p<MVE_VQMOVNu32th,  1, 1, 1, MVE_v8i16, MVE_v4i32>;
4934defm : MVE_VQMOVN_p<MVE_VQMOVNu16bh,  1, 1, 0, MVE_v16i8, MVE_v8i16>;
4935defm : MVE_VQMOVN_p<MVE_VQMOVNu16th,  1, 1, 1, MVE_v16i8, MVE_v8i16>;
4936defm : MVE_VQMOVN_p<MVE_VQMOVUNs32bh, 1, 0, 0, MVE_v8i16, MVE_v4i32>;
4937defm : MVE_VQMOVN_p<MVE_VQMOVUNs32th, 1, 0, 1, MVE_v8i16, MVE_v4i32>;
4938defm : MVE_VQMOVN_p<MVE_VQMOVUNs16bh, 1, 0, 0, MVE_v16i8, MVE_v8i16>;
4939defm : MVE_VQMOVN_p<MVE_VQMOVUNs16th, 1, 0, 1, MVE_v16i8, MVE_v8i16>;
4940
4941def SDTARMVMOVNQ : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0, 1>,
4942                                        SDTCisVec<2>, SDTCisVT<3, i32>]>;
4943def MVEvqmovns   : SDNode<"ARMISD::VQMOVNs", SDTARMVMOVNQ>;
4944def MVEvqmovnu   : SDNode<"ARMISD::VQMOVNu", SDTARMVMOVNQ>;
4945
4946let Predicates = [HasMVEInt] in {
4947  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 0))),
4948            (v8i16 (MVE_VQMOVNs32bh (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4949  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 1))),
4950            (v8i16 (MVE_VQMOVNs32th (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4951  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 0))),
4952            (v16i8 (MVE_VQMOVNs16bh (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4953  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 1))),
4954            (v16i8 (MVE_VQMOVNs16th (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4955
4956  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 0))),
4957            (v8i16 (MVE_VQMOVNu32bh (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4958  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), (i32 1))),
4959            (v8i16 (MVE_VQMOVNu32th (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm)))>;
4960  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 0))),
4961            (v16i8 (MVE_VQMOVNu16bh (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4962  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), (i32 1))),
4963            (v16i8 (MVE_VQMOVNu16th (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm)))>;
4964
4965  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshrsImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 0))),
4966            (v8i16 (MVE_VQSHRNbhs32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4967  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshrsImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 0))),
4968            (v16i8 (MVE_VQSHRNbhs16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4969  def : Pat<(v8i16 (MVEvqmovns (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshrsImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 1))),
4970            (v8i16 (MVE_VQSHRNths32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4971  def : Pat<(v16i8 (MVEvqmovns (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshrsImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 1))),
4972            (v16i8 (MVE_VQSHRNths16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4973
4974  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshruImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 0))),
4975            (v8i16 (MVE_VQSHRNbhu32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4976  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshruImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 0))),
4977            (v16i8 (MVE_VQSHRNbhu16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4978  def : Pat<(v8i16 (MVEvqmovnu (v8i16 MQPR:$Qd_src), (v4i32 (ARMvshruImm (v4i32 MQPR:$Qm), imm0_31:$imm)), (i32 1))),
4979            (v8i16 (MVE_VQSHRNthu32 (v8i16 MQPR:$Qd_src), (v4i32 MQPR:$Qm), imm0_31:$imm))>;
4980  def : Pat<(v16i8 (MVEvqmovnu (v16i8 MQPR:$Qd_src), (v8i16 (ARMvshruImm (v8i16 MQPR:$Qm), imm0_15:$imm)), (i32 1))),
4981            (v16i8 (MVE_VQSHRNthu16 (v16i8 MQPR:$Qd_src), (v8i16 MQPR:$Qm), imm0_15:$imm))>;
4982}
4983
4984class MVE_VCVT_ff<string iname, string suffix, bit op, bit T,
4985                  dag iops_extra, vpred_ops vpred, string cstr>
4986  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
4987                   !con(iops_extra, (ins MQPR:$Qm)), "$Qd, $Qm",
4988                   vpred, cstr, 0b10, []> {
4989  let Inst{28} = op;
4990  let Inst{21-16} = 0b111111;
4991  let Inst{12} = T;
4992  let Inst{8-7} = 0b00;
4993  let Inst{0} = 0b1;
4994
4995  let Predicates = [HasMVEFloat];
4996  let retainsPreviousHalfElement = 1;
4997}
4998
4999def SDTARMVCVTL    : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>,
5000                                         SDTCisVT<2, i32>]>;
5001def MVEvcvtn       : SDNode<"ARMISD::VCVTN", SDTARMVMOVNQ>;
5002def MVEvcvtl       : SDNode<"ARMISD::VCVTL", SDTARMVCVTL>;
5003
5004multiclass MVE_VCVT_f2h_m<string iname, int half> {
5005  def "": MVE_VCVT_ff<iname, "f16.f32", 0b0, half,
5006                      (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
5007  defvar Inst = !cast<Instruction>(NAME);
5008
5009  let Predicates = [HasMVEFloat] in {
5010    def : Pat<(v8f16 (int_arm_mve_vcvt_narrow
5011                         (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),
5012              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;
5013    def : Pat<(v8f16 (int_arm_mve_vcvt_narrow_predicated
5014                         (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half),
5015                         (v4i1 VCCR:$mask))),
5016              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm),
5017                           ARMVCCThen, (v4i1 VCCR:$mask), zero_reg))>;
5018
5019    def : Pat<(v8f16 (MVEvcvtn (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm), (i32 half))),
5020              (v8f16 (Inst (v8f16 MQPR:$Qd_src), (v4f32 MQPR:$Qm)))>;
5021  }
5022}
5023
5024multiclass MVE_VCVT_h2f_m<string iname, int half> {
5025  def "": MVE_VCVT_ff<iname, "f32.f16", 0b1, half, (ins), vpred_r, "">;
5026  defvar Inst = !cast<Instruction>(NAME);
5027
5028  let Predicates = [HasMVEFloat] in {
5029    def : Pat<(v4f32 (int_arm_mve_vcvt_widen (v8f16 MQPR:$Qm), (i32 half))),
5030              (v4f32 (Inst (v8f16 MQPR:$Qm)))>;
5031    def : Pat<(v4f32 (int_arm_mve_vcvt_widen_predicated
5032                         (v4f32 MQPR:$inactive), (v8f16 MQPR:$Qm), (i32 half),
5033                         (v4i1 VCCR:$mask))),
5034              (v4f32 (Inst (v8f16 MQPR:$Qm), ARMVCCThen,
5035                           (v4i1 VCCR:$mask), zero_reg, (v4f32 MQPR:$inactive)))>;
5036
5037    def : Pat<(v4f32 (MVEvcvtl (v8f16 MQPR:$Qm), (i32 half))),
5038              (v4f32 (Inst (v8f16 MQPR:$Qm)))>;
5039  }
5040}
5041
5042defm MVE_VCVTf16f32bh : MVE_VCVT_f2h_m<"vcvtb", 0b0>;
5043defm MVE_VCVTf16f32th : MVE_VCVT_f2h_m<"vcvtt", 0b1>;
5044defm MVE_VCVTf32f16bh : MVE_VCVT_h2f_m<"vcvtb", 0b0>;
5045defm MVE_VCVTf32f16th : MVE_VCVT_h2f_m<"vcvtt", 0b1>;
5046
5047class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
5048                 string cstr="">
5049  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
5050                   (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
5051                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, size, []> {
5052  bits<4> Qn;
5053  bit rot;
5054
5055  let Inst{28} = halve;
5056  let Inst{21-20} = size;
5057  let Inst{19-17} = Qn{2-0};
5058  let Inst{16} = 0b0;
5059  let Inst{12} = rot;
5060  let Inst{8} = 0b1;
5061  let Inst{7} = Qn{3};
5062  let Inst{0} = 0b0;
5063}
5064
5065multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,
5066                        bit halve, string cstr=""> {
5067  def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;
5068  defvar Inst = !cast<Instruction>(NAME);
5069
5070  let Predicates = [HasMVEInt] in {
5071    def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,
5072                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
5073              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5074                             imm:$rot))>;
5075
5076    def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,
5077                            imm:$rot, (VTI.Vec MQPR:$inactive),
5078                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5079                            (VTI.Pred VCCR:$mask))),
5080              (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5081                             imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
5082                             (VTI.Vec MQPR:$inactive)))>;
5083
5084  }
5085}
5086
5087defm MVE_VCADDi8   : MVE_VxCADD_m<"vcadd", MVE_v16i8, 0b1>;
5088defm MVE_VCADDi16  : MVE_VxCADD_m<"vcadd", MVE_v8i16, 0b1>;
5089defm MVE_VCADDi32  : MVE_VxCADD_m<"vcadd", MVE_v4i32, 0b1, "@earlyclobber $Qd">;
5090
5091defm MVE_VHCADDs8  : MVE_VxCADD_m<"vhcadd", MVE_v16s8, 0b0>;
5092defm MVE_VHCADDs16 : MVE_VxCADD_m<"vhcadd", MVE_v8s16, 0b0>;
5093defm MVE_VHCADDs32 : MVE_VxCADD_m<"vhcadd", MVE_v4s32, 0b0, "@earlyclobber $Qd">;
5094
5095class MVE_VADCSBC<string iname, bit I, bit subtract,
5096                  dag carryin, list<dag> pattern=[]>
5097  : MVE_qDest_qSrc<iname, "i32", (outs MQPR:$Qd, cl_FPSCR_NZCV:$carryout),
5098                   !con((ins MQPR:$Qn, MQPR:$Qm), carryin),
5099                   "$Qd, $Qn, $Qm", vpred_r, "", 0b10, pattern> {
5100  bits<4> Qn;
5101
5102  let Inst{28} = subtract;
5103  let Inst{21-20} = 0b11;
5104  let Inst{19-17} = Qn{2-0};
5105  let Inst{16} = 0b0;
5106  let Inst{12} = I;
5107  let Inst{8} = 0b1;
5108  let Inst{7} = Qn{3};
5109  let Inst{0} = 0b0;
5110
5111  // Custom decoder method in order to add the FPSCR operand(s), which
5112  // Tablegen won't do right
5113  let DecoderMethod = "DecodeMVEVADCInstruction";
5114}
5115
5116def MVE_VADC  : MVE_VADCSBC<"vadc",  0b0, 0b0, (ins cl_FPSCR_NZCV:$carryin)>;
5117def MVE_VADCI : MVE_VADCSBC<"vadci", 0b1, 0b0, (ins)>;
5118
5119def MVE_VSBC  : MVE_VADCSBC<"vsbc",  0b0, 0b1, (ins cl_FPSCR_NZCV:$carryin)>;
5120def MVE_VSBCI : MVE_VADCSBC<"vsbci", 0b1, 0b1, (ins)>;
5121
5122class MVE_VQDMULL<string iname, string suffix, bit size, bit T,
5123                  string cstr="", list<dag> pattern=[]>
5124  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
5125                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
5126                   vpred_r, cstr, !if(size, 0b10, 0b01), pattern> {
5127  bits<4> Qn;
5128
5129  let Inst{28} = size;
5130  let Inst{21-20} = 0b11;
5131  let Inst{19-17} = Qn{2-0};
5132  let Inst{16} = 0b0;
5133  let Inst{12} = T;
5134  let Inst{8} = 0b1;
5135  let Inst{7} = Qn{3};
5136  let Inst{0} = 0b1;
5137  let validForTailPredication = 1;
5138  let doubleWidthResult = 1;
5139}
5140
5141multiclass MVE_VQDMULL_m<string iname, MVEVectorVTInfo VTI, bit size, bit T,
5142                         string cstr> {
5143  def "" : MVE_VQDMULL<iname, VTI.Suffix, size, T, cstr>;
5144  defvar Inst = !cast<Instruction>(NAME);
5145
5146  let Predicates = [HasMVEInt] in {
5147    // Unpredicated saturating multiply
5148    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5149                                               (VTI.Vec MQPR:$Qn), (i32 T))),
5150              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
5151    // Predicated saturating multiply
5152    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5153                                    (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
5154                                    (i32 T), (VTI.DblPred VCCR:$mask),
5155                                    (VTI.DblVec MQPR:$inactive))),
5156              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
5157                                ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
5158                                (VTI.DblVec MQPR:$inactive)))>;
5159  }
5160}
5161
5162multiclass MVE_VQDMULL_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5163  defm bh : MVE_VQDMULL_m<"vqdmullb", VTI, size, 0b0, cstr>;
5164  defm th : MVE_VQDMULL_m<"vqdmullt", VTI, size, 0b1, cstr>;
5165}
5166
5167defm MVE_VQDMULLs16 : MVE_VQDMULL_halves<MVE_v8s16, 0b0>;
5168defm MVE_VQDMULLs32 : MVE_VQDMULL_halves<MVE_v4s32, 0b1, "@earlyclobber $Qd">;
5169
5170// end of mve_qDest_qSrc
5171
5172// start of mve_qDest_rSrc
5173
5174class MVE_qr_base<dag oops, dag iops, string iname, string suffix, string ops,
5175                  vpred_ops vpred, string cstr, bits<2> vecsize, list<dag> pattern=[]>
5176   : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, vecsize, pattern> {
5177  bits<4> Qd;
5178  bits<4> Qn;
5179  bits<4> Rm;
5180
5181  let Inst{25-23} = 0b100;
5182  let Inst{22} = Qd{3};
5183  let Inst{19-17} = Qn{2-0};
5184  let Inst{15-13} = Qd{2-0};
5185  let Inst{11-9} = 0b111;
5186  let Inst{7} = Qn{3};
5187  let Inst{6} = 0b1;
5188  let Inst{4} = 0b0;
5189  let Inst{3-0} = Rm{3-0};
5190}
5191
5192class MVE_qDest_rSrc<string iname, string suffix, string cstr="", bits<2> vecsize, list<dag> pattern=[]>
5193  : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qn, rGPR:$Rm),
5194                 iname, suffix, "$Qd, $Qn, $Rm", vpred_r, cstr,
5195                 vecsize, pattern>;
5196
5197class MVE_qDestSrc_rSrc<string iname, string suffix, bits<2> vecsize, list<dag> pattern=[]>
5198  : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qn, rGPR:$Rm),
5199                 iname, suffix, "$Qd, $Qn, $Rm", vpred_n, "$Qd = $Qd_src",
5200                 vecsize, pattern>;
5201
5202class MVE_qDest_single_rSrc<string iname, string suffix, bits<2> vecsize, list<dag> pattern=[]>
5203  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, rGPR:$Rm), NoItinerary, iname,
5204          suffix, "$Qd, $Rm", vpred_n, "$Qd = $Qd_src", vecsize, pattern> {
5205  bits<4> Qd;
5206  bits<4> Rm;
5207
5208  let Inst{22} = Qd{3};
5209  let Inst{15-13} = Qd{2-0};
5210  let Inst{3-0} = Rm{3-0};
5211}
5212
5213// Patterns for vector-scalar instructions with integer operands
5214multiclass MVE_vec_scalar_int_pat_m<Instruction inst, MVEVectorVTInfo VTI,
5215                                    SDPatternOperator unpred_op,
5216                                    SDPatternOperator pred_op,
5217                                    bit unpred_has_sign = 0,
5218                                    bit pred_has_sign = 0> {
5219  defvar UnpredSign = !if(unpred_has_sign, (? (i32 VTI.Unsigned)), (?));
5220  defvar PredSign = !if(pred_has_sign, (? (i32 VTI.Unsigned)), (?));
5221
5222  let Predicates = [HasMVEInt] in {
5223    // Unpredicated version
5224    def : Pat<(VTI.Vec !con((unpred_op (VTI.Vec MQPR:$Qm),
5225                                       (VTI.Vec (ARMvdup rGPR:$val))),
5226                            UnpredSign)),
5227              (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5228    // Predicated version
5229    def : Pat<(VTI.Vec !con((pred_op (VTI.Vec MQPR:$Qm),
5230                                     (VTI.Vec (ARMvdup rGPR:$val))),
5231                            PredSign,
5232                            (pred_op (VTI.Pred VCCR:$mask),
5233                                     (VTI.Vec MQPR:$inactive)))),
5234              (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5235                             ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
5236                             (VTI.Vec MQPR:$inactive)))>;
5237  }
5238}
5239
5240class MVE_VADDSUB_qr<string iname, string suffix, bits<2> size,
5241                     bit bit_5, bit bit_12, bit bit_16, bit bit_28>
5242  : MVE_qDest_rSrc<iname, suffix, "", size> {
5243
5244  let Inst{28} = bit_28;
5245  let Inst{21-20} = size;
5246  let Inst{16} = bit_16;
5247  let Inst{12} = bit_12;
5248  let Inst{8} = 0b1;
5249  let Inst{5} = bit_5;
5250  let validForTailPredication = 1;
5251}
5252
5253// Vector-scalar add/sub
5254multiclass MVE_VADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5255                            SDNode Op, Intrinsic PredInt> {
5256  def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b0, subtract, 0b1, 0b0>;
5257  let Predicates = [HasMVEInt] in {
5258    defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
5259  }
5260}
5261
5262multiclass MVE_VADD_qr_m<MVEVectorVTInfo VTI>
5263  : MVE_VADDSUB_qr_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
5264
5265multiclass MVE_VSUB_qr_m<MVEVectorVTInfo VTI>
5266  : MVE_VADDSUB_qr_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
5267
5268defm MVE_VADD_qr_i8  : MVE_VADD_qr_m<MVE_v16i8>;
5269defm MVE_VADD_qr_i16 : MVE_VADD_qr_m<MVE_v8i16>;
5270defm MVE_VADD_qr_i32 : MVE_VADD_qr_m<MVE_v4i32>;
5271
5272defm MVE_VSUB_qr_i8  : MVE_VSUB_qr_m<MVE_v16i8>;
5273defm MVE_VSUB_qr_i16 : MVE_VSUB_qr_m<MVE_v8i16>;
5274defm MVE_VSUB_qr_i32 : MVE_VSUB_qr_m<MVE_v4i32>;
5275
5276// Vector-scalar saturating add/sub
5277multiclass MVE_VQADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5278                             SDNode Op, Intrinsic PredInt> {
5279  def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b1, subtract,
5280                          0b0, VTI.Unsigned>;
5281
5282  let Predicates = [HasMVEInt] in {
5283    defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
5284                               !cast<Instruction>(NAME)>;
5285  }
5286}
5287
5288multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI, SDNode Op>
5289  : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, Op, int_arm_mve_qadd_predicated>;
5290
5291multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI, SDNode Op>
5292  : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, Op, int_arm_mve_qsub_predicated>;
5293
5294defm MVE_VQADD_qr_s8  : MVE_VQADD_qr_m<MVE_v16s8, saddsat>;
5295defm MVE_VQADD_qr_s16 : MVE_VQADD_qr_m<MVE_v8s16, saddsat>;
5296defm MVE_VQADD_qr_s32 : MVE_VQADD_qr_m<MVE_v4s32, saddsat>;
5297defm MVE_VQADD_qr_u8  : MVE_VQADD_qr_m<MVE_v16u8, uaddsat>;
5298defm MVE_VQADD_qr_u16 : MVE_VQADD_qr_m<MVE_v8u16, uaddsat>;
5299defm MVE_VQADD_qr_u32 : MVE_VQADD_qr_m<MVE_v4u32, uaddsat>;
5300
5301defm MVE_VQSUB_qr_s8  : MVE_VQSUB_qr_m<MVE_v16s8, ssubsat>;
5302defm MVE_VQSUB_qr_s16 : MVE_VQSUB_qr_m<MVE_v8s16, ssubsat>;
5303defm MVE_VQSUB_qr_s32 : MVE_VQSUB_qr_m<MVE_v4s32, ssubsat>;
5304defm MVE_VQSUB_qr_u8  : MVE_VQSUB_qr_m<MVE_v16u8, usubsat>;
5305defm MVE_VQSUB_qr_u16 : MVE_VQSUB_qr_m<MVE_v8u16, usubsat>;
5306defm MVE_VQSUB_qr_u32 : MVE_VQSUB_qr_m<MVE_v4u32, usubsat>;
5307
5308class MVE_VQDMULL_qr<string iname, string suffix, bit size,
5309                     bit T, string cstr="", list<dag> pattern=[]>
5310  : MVE_qDest_rSrc<iname, suffix, cstr, !if(size, 0b10, 0b01), pattern> {
5311
5312  let Inst{28} = size;
5313  let Inst{21-20} = 0b11;
5314  let Inst{16} = 0b0;
5315  let Inst{12} = T;
5316  let Inst{8} = 0b1;
5317  let Inst{5} = 0b1;
5318  let validForTailPredication = 1;
5319  let doubleWidthResult = 1;
5320}
5321
5322multiclass MVE_VQDMULL_qr_m<string iname, MVEVectorVTInfo VTI, bit size,
5323                            bit T, string cstr> {
5324  def "" : MVE_VQDMULL_qr<iname, VTI.Suffix, size, T, cstr>;
5325  defvar Inst = !cast<Instruction>(NAME);
5326
5327  let Predicates = [HasMVEInt] in {
5328    // Unpredicated saturating multiply
5329    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5330                                               (VTI.Vec (ARMvdup rGPR:$val)),
5331                                               (i32 T))),
5332              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5333    // Predicated saturating multiply
5334    def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5335                                    (VTI.Vec MQPR:$Qm),
5336                                    (VTI.Vec (ARMvdup rGPR:$val)),
5337                                    (i32 T),
5338                                    (VTI.DblPred VCCR:$mask),
5339                                    (VTI.DblVec MQPR:$inactive))),
5340              (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5341                             ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
5342                             (VTI.DblVec MQPR:$inactive)))>;
5343  }
5344}
5345
5346multiclass MVE_VQDMULL_qr_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5347  defm bh : MVE_VQDMULL_qr_m<"vqdmullb", VTI, size, 0b0, cstr>;
5348  defm th : MVE_VQDMULL_qr_m<"vqdmullt", VTI, size, 0b1, cstr>;
5349}
5350
5351defm MVE_VQDMULL_qr_s16 : MVE_VQDMULL_qr_halves<MVE_v8s16, 0b0>;
5352defm MVE_VQDMULL_qr_s32 : MVE_VQDMULL_qr_halves<MVE_v4s32, 0b1, "@earlyclobber $Qd">;
5353
5354class MVE_VxADDSUB_qr<string iname, string suffix,
5355                      bit bit_28, bits<2> size, bit subtract,
5356                      bits<2> vecsize, list<dag> pattern=[]>
5357  : MVE_qDest_rSrc<iname, suffix, "", vecsize, pattern> {
5358
5359  let Inst{28} = bit_28;
5360  let Inst{21-20} = size;
5361  let Inst{16} = 0b0;
5362  let Inst{12} = subtract;
5363  let Inst{8} = 0b1;
5364  let Inst{5} = 0b0;
5365  let validForTailPredication = 1;
5366}
5367
5368multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract, SDNode Op,
5369                             Intrinsic unpred_int, Intrinsic pred_int, PatFrag add_op, PatFrag shift_op> {
5370  def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>;
5371  defm : MVE_TwoOpPatternDup<VTI, Op, pred_int, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
5372  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME),
5373                                  VTI, unpred_int, pred_int, 1, 1>;
5374  defvar Inst = !cast<Instruction>(NAME);
5375
5376  let Predicates = [HasMVEInt] in {
5377    def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))),
5378              (Inst MQPR:$Qm, rGPR:$Rn)>;
5379  }
5380}
5381
5382multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op, SDNode Op> :
5383  MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, Op, int_arm_mve_vhadd,
5384                    int_arm_mve_hadd_predicated, add_op, shift_op>;
5385
5386multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :
5387  MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, null_frag, int_arm_mve_vhsub,
5388                    int_arm_mve_hsub_predicated, add_op, shift_op>;
5389
5390defm MVE_VHADD_qr_s8  : MVE_VHADD_qr_m<MVE_v16s8, addnsw, ARMvshrsImm, avgfloors>;
5391defm MVE_VHADD_qr_s16 : MVE_VHADD_qr_m<MVE_v8s16, addnsw, ARMvshrsImm, avgfloors>;
5392defm MVE_VHADD_qr_s32 : MVE_VHADD_qr_m<MVE_v4s32, addnsw, ARMvshrsImm, avgfloors>;
5393defm MVE_VHADD_qr_u8  : MVE_VHADD_qr_m<MVE_v16u8, addnuw, ARMvshruImm, avgflooru>;
5394defm MVE_VHADD_qr_u16 : MVE_VHADD_qr_m<MVE_v8u16, addnuw, ARMvshruImm, avgflooru>;
5395defm MVE_VHADD_qr_u32 : MVE_VHADD_qr_m<MVE_v4u32, addnuw, ARMvshruImm, avgflooru>;
5396
5397defm MVE_VHSUB_qr_s8  : MVE_VHSUB_qr_m<MVE_v16s8, subnsw, ARMvshrsImm>;
5398defm MVE_VHSUB_qr_s16 : MVE_VHSUB_qr_m<MVE_v8s16, subnsw, ARMvshrsImm>;
5399defm MVE_VHSUB_qr_s32 : MVE_VHSUB_qr_m<MVE_v4s32, subnsw, ARMvshrsImm>;
5400defm MVE_VHSUB_qr_u8  : MVE_VHSUB_qr_m<MVE_v16u8, subnuw, ARMvshruImm>;
5401defm MVE_VHSUB_qr_u16 : MVE_VHSUB_qr_m<MVE_v8u16, subnuw, ARMvshruImm>;
5402defm MVE_VHSUB_qr_u32 : MVE_VHSUB_qr_m<MVE_v4u32, subnuw, ARMvshruImm>;
5403
5404multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract,
5405                            SDNode Op, Intrinsic PredInt, SDPatternOperator IdentityVec> {
5406  def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, subtract, VTI.Size>;
5407  defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ),
5408                              !cast<Instruction>(NAME), IdentityVec>;
5409}
5410
5411let Predicates = [HasMVEFloat] in {
5412  defm MVE_VADD_qr_f32 : MVE_VADDSUB_qr_f<"vadd", MVE_v4f32, 0b0, fadd,
5413                                          int_arm_mve_add_predicated, ARMimmMinusZeroF>;
5414  defm MVE_VADD_qr_f16 : MVE_VADDSUB_qr_f<"vadd", MVE_v8f16, 0b0, fadd,
5415                                          int_arm_mve_add_predicated, ARMimmMinusZeroH>;
5416
5417  defm MVE_VSUB_qr_f32 : MVE_VADDSUB_qr_f<"vsub", MVE_v4f32, 0b1, fsub,
5418                                          int_arm_mve_sub_predicated, ARMimmAllZerosV>;
5419  defm MVE_VSUB_qr_f16 : MVE_VADDSUB_qr_f<"vsub", MVE_v8f16, 0b1, fsub,
5420                                          int_arm_mve_sub_predicated, ARMimmAllZerosV>;
5421}
5422
5423class MVE_VxSHL_qr<string iname, string suffix, bit U, bits<2> size,
5424                   bit bit_7, bit bit_17, list<dag> pattern=[]>
5425  : MVE_qDest_single_rSrc<iname, suffix, size, pattern> {
5426
5427  let Inst{28} = U;
5428  let Inst{25-23} = 0b100;
5429  let Inst{21-20} = 0b11;
5430  let Inst{19-18} = size;
5431  let Inst{17} = bit_17;
5432  let Inst{16} = 0b1;
5433  let Inst{12-8} = 0b11110;
5434  let Inst{7} = bit_7;
5435  let Inst{6-4} = 0b110;
5436  let validForTailPredication = 1;
5437}
5438
5439multiclass MVE_VxSHL_qr_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
5440  def "" : MVE_VxSHL_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
5441  defvar Inst = !cast<Instruction>(NAME);
5442
5443  def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar
5444                         (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5445                         (i32 q), (i32 r), (i32 VTI.Unsigned))),
5446            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh)))>;
5447
5448  def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar_predicated
5449                         (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5450                         (i32 q), (i32 r), (i32 VTI.Unsigned),
5451                         (VTI.Pred VCCR:$mask))),
5452            (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5453                           ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
5454}
5455
5456multiclass MVE_VxSHL_qr_types<string iname, bit bit_7, bit bit_17> {
5457  defm s8  : MVE_VxSHL_qr_p<iname, MVE_v16s8, bit_7, bit_17>;
5458  defm s16 : MVE_VxSHL_qr_p<iname, MVE_v8s16, bit_7, bit_17>;
5459  defm s32 : MVE_VxSHL_qr_p<iname, MVE_v4s32, bit_7, bit_17>;
5460  defm u8  : MVE_VxSHL_qr_p<iname, MVE_v16u8, bit_7, bit_17>;
5461  defm u16 : MVE_VxSHL_qr_p<iname, MVE_v8u16, bit_7, bit_17>;
5462  defm u32 : MVE_VxSHL_qr_p<iname, MVE_v4u32, bit_7, bit_17>;
5463}
5464
5465defm MVE_VSHL_qr   : MVE_VxSHL_qr_types<"vshl",   0b0, 0b0>;
5466defm MVE_VRSHL_qr  : MVE_VxSHL_qr_types<"vrshl",  0b0, 0b1>;
5467defm MVE_VQSHL_qr  : MVE_VxSHL_qr_types<"vqshl",  0b1, 0b0>;
5468defm MVE_VQRSHL_qr : MVE_VxSHL_qr_types<"vqrshl", 0b1, 0b1>;
5469
5470let Predicates = [HasMVEInt] in {
5471  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 (ARMvdup rGPR:$Rm)))),
5472            (v4i32 (MVE_VSHL_qru32 (v4i32 MQPR:$Qm), rGPR:$Rm))>;
5473  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 (ARMvdup rGPR:$Rm)))),
5474            (v8i16 (MVE_VSHL_qru16 (v8i16 MQPR:$Qm), rGPR:$Rm))>;
5475  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 (ARMvdup rGPR:$Rm)))),
5476            (v16i8 (MVE_VSHL_qru8 (v16i8 MQPR:$Qm), rGPR:$Rm))>;
5477
5478  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 (ARMvdup rGPR:$Rm)))),
5479            (v4i32 (MVE_VSHL_qrs32 (v4i32 MQPR:$Qm), rGPR:$Rm))>;
5480  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 (ARMvdup rGPR:$Rm)))),
5481            (v8i16 (MVE_VSHL_qrs16 (v8i16 MQPR:$Qm), rGPR:$Rm))>;
5482  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 (ARMvdup rGPR:$Rm)))),
5483            (v16i8 (MVE_VSHL_qrs8 (v16i8 MQPR:$Qm), rGPR:$Rm))>;
5484}
5485
5486class MVE_VBRSR<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
5487  : MVE_qDest_rSrc<iname, suffix, "", size, pattern> {
5488
5489  let Inst{28} = 0b1;
5490  let Inst{21-20} = size;
5491  let Inst{16} = 0b1;
5492  let Inst{12} = 0b1;
5493  let Inst{8} = 0b0;
5494  let Inst{5} = 0b1;
5495  let validForTailPredication = 1;
5496}
5497
5498def MVE_VBRSR8  : MVE_VBRSR<"vbrsr", "8", 0b00>;
5499def MVE_VBRSR16 : MVE_VBRSR<"vbrsr", "16", 0b01>;
5500def MVE_VBRSR32 : MVE_VBRSR<"vbrsr", "32", 0b10>;
5501
5502multiclass MVE_VBRSR_pat_m<MVEVectorVTInfo VTI, Instruction Inst> {
5503  // Unpredicated
5504  def : Pat<(VTI.Vec (int_arm_mve_vbrsr (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm))),
5505            (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm)))>;
5506  // Predicated
5507  def : Pat<(VTI.Vec (int_arm_mve_vbrsr_predicated
5508                          (VTI.Vec MQPR:$inactive),
5509                          (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5510                          (VTI.Pred VCCR:$mask))),
5511            (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5512                          ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
5513                          (VTI.Vec MQPR:$inactive)))>;
5514}
5515
5516let Predicates = [HasMVEInt] in {
5517  def : Pat<(v16i8 ( bitreverse (v16i8 MQPR:$val1))),
5518            (v16i8 ( MVE_VBRSR8 (v16i8 MQPR:$val1), (t2MOVi (i32 8)) ))>;
5519
5520  def : Pat<(v4i32 ( bitreverse (v4i32 MQPR:$val1))),
5521            (v4i32 ( MVE_VBRSR32 (v4i32 MQPR:$val1), (t2MOVi (i32 32)) ))>;
5522
5523  def : Pat<(v8i16 ( bitreverse (v8i16 MQPR:$val1))),
5524            (v8i16 ( MVE_VBRSR16 (v8i16 MQPR:$val1), (t2MOVi (i32 16)) ))>;
5525
5526  defm : MVE_VBRSR_pat_m<MVE_v16i8, MVE_VBRSR8>;
5527  defm : MVE_VBRSR_pat_m<MVE_v8i16, MVE_VBRSR16>;
5528  defm : MVE_VBRSR_pat_m<MVE_v4i32, MVE_VBRSR32>;
5529}
5530
5531let Predicates = [HasMVEFloat] in {
5532  defm : MVE_VBRSR_pat_m<MVE_v8f16, MVE_VBRSR16>;
5533  defm : MVE_VBRSR_pat_m<MVE_v4f32, MVE_VBRSR32>;
5534}
5535
5536class MVE_VMUL_qr_int<string iname, string suffix, bits<2> size>
5537  : MVE_qDest_rSrc<iname, suffix, "", size> {
5538
5539  let Inst{28} = 0b0;
5540  let Inst{21-20} = size;
5541  let Inst{16} = 0b1;
5542  let Inst{12} = 0b1;
5543  let Inst{8} = 0b0;
5544  let Inst{5} = 0b1;
5545  let validForTailPredication = 1;
5546}
5547
5548multiclass MVE_VMUL_qr_int_m<MVEVectorVTInfo VTI> {
5549  def "" : MVE_VMUL_qr_int<"vmul", VTI.Suffix, VTI.Size>;
5550  let Predicates = [HasMVEInt] in {
5551    defm : MVE_TwoOpPatternDup<VTI, mul, int_arm_mve_mul_predicated, (? ),
5552                               !cast<Instruction>(NAME), ARMimmOneV>;
5553  }
5554}
5555
5556defm MVE_VMUL_qr_i8  : MVE_VMUL_qr_int_m<MVE_v16i8>;
5557defm MVE_VMUL_qr_i16 : MVE_VMUL_qr_int_m<MVE_v8i16>;
5558defm MVE_VMUL_qr_i32 : MVE_VMUL_qr_int_m<MVE_v4i32>;
5559
5560class MVE_VxxMUL_qr<string iname, string suffix,
5561                    bit bit_28, bits<2> size, bits<2> vecsize, list<dag> pattern=[]>
5562  : MVE_qDest_rSrc<iname, suffix, "", vecsize, pattern> {
5563
5564  let Inst{28} = bit_28;
5565  let Inst{21-20} = size;
5566  let Inst{16} = 0b1;
5567  let Inst{12} = 0b0;
5568  let Inst{8} = 0b0;
5569  let Inst{5} = 0b1;
5570  let validForTailPredication = 1;
5571}
5572
5573multiclass MVE_VxxMUL_qr_m<string iname, MVEVectorVTInfo VTI, bit bit_28,
5574                           PatFrag Op, Intrinsic int_unpred, Intrinsic int_pred> {
5575  def "" : MVE_VxxMUL_qr<iname, VTI.Suffix, bit_28, VTI.Size, VTI.Size>;
5576
5577  let Predicates = [HasMVEInt] in {
5578    defm : MVE_TwoOpPatternDup<VTI, Op, int_pred, (? ), !cast<Instruction>(NAME)>;
5579  }
5580  defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI, int_unpred, int_pred>;
5581}
5582
5583multiclass MVE_VQDMULH_qr_m<MVEVectorVTInfo VTI> :
5584  MVE_VxxMUL_qr_m<"vqdmulh", VTI, 0b0, MVEvqdmulh,
5585                  int_arm_mve_vqdmulh, int_arm_mve_qdmulh_predicated>;
5586
5587multiclass MVE_VQRDMULH_qr_m<MVEVectorVTInfo VTI> :
5588  MVE_VxxMUL_qr_m<"vqrdmulh", VTI, 0b1, null_frag,
5589                  int_arm_mve_vqrdmulh, int_arm_mve_qrdmulh_predicated>;
5590
5591defm MVE_VQDMULH_qr_s8    : MVE_VQDMULH_qr_m<MVE_v16s8>;
5592defm MVE_VQDMULH_qr_s16   : MVE_VQDMULH_qr_m<MVE_v8s16>;
5593defm MVE_VQDMULH_qr_s32   : MVE_VQDMULH_qr_m<MVE_v4s32>;
5594
5595defm MVE_VQRDMULH_qr_s8   : MVE_VQRDMULH_qr_m<MVE_v16s8>;
5596defm MVE_VQRDMULH_qr_s16  : MVE_VQRDMULH_qr_m<MVE_v8s16>;
5597defm MVE_VQRDMULH_qr_s32  : MVE_VQRDMULH_qr_m<MVE_v4s32>;
5598
5599multiclass MVE_VxxMUL_qr_f_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec> {
5600  let validForTailPredication = 1 in
5601  def "" : MVE_VxxMUL_qr<"vmul", VTI.Suffix, VTI.Size{0}, 0b11, VTI.Size>;
5602  defm : MVE_TwoOpPatternDup<VTI, fmul, int_arm_mve_mul_predicated, (? ),
5603                             !cast<Instruction>(NAME), IdentityVec>;
5604}
5605
5606let Predicates = [HasMVEFloat] in {
5607  defm MVE_VMUL_qr_f16   : MVE_VxxMUL_qr_f_m<MVE_v8f16, ARMimmOneH>;
5608  defm MVE_VMUL_qr_f32   : MVE_VxxMUL_qr_f_m<MVE_v4f32, ARMimmOneF>;
5609}
5610
5611class MVE_VFMAMLA_qr<string iname, string suffix,
5612                     bit bit_28, bits<2> bits_21_20, bit S,
5613                     bits<2> vecsize, list<dag> pattern=[]>
5614  : MVE_qDestSrc_rSrc<iname, suffix, vecsize, pattern> {
5615
5616  let Inst{28} = bit_28;
5617  let Inst{21-20} = bits_21_20;
5618  let Inst{16} = 0b1;
5619  let Inst{12} = S;
5620  let Inst{8} = 0b0;
5621  let Inst{5} = 0b0;
5622  let validForTailPredication = 1;
5623  let hasSideEffects = 0;
5624}
5625
5626multiclass MVE_VMLA_qr_multi<string iname, MVEVectorVTInfo VTI,
5627                             bit scalar_addend> {
5628  def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
5629                         scalar_addend, VTI.Size>;
5630  defvar Inst = !cast<Instruction>(NAME);
5631  defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # iname # "_n_predicated");
5632  defvar v1   = (VTI.Vec MQPR:$v1);
5633  defvar v2   = (VTI.Vec MQPR:$v2);
5634  defvar vs   = (VTI.Vec (ARMvdup rGPR:$s));
5635  defvar s    = (i32 rGPR:$s);
5636  defvar pred = (VTI.Pred VCCR:$pred);
5637
5638  // The signed and unsigned variants of this instruction have different
5639  // encodings, but they're functionally identical. For the sake of
5640  // determinism, we generate only the unsigned variant.
5641  if VTI.Unsigned then let Predicates = [HasMVEInt] in {
5642    if scalar_addend then {
5643      def : Pat<(VTI.Vec (add (mul v1, v2), vs)),
5644                (VTI.Vec (Inst v1, v2, s))>;
5645    } else {
5646      def : Pat<(VTI.Vec (add (mul v2, vs), v1)),
5647                (VTI.Vec (Inst v1, v2, s))>;
5648    }
5649
5650    def : Pat<(VTI.Vec (pred_int v1, v2, s, pred)),
5651              (VTI.Vec (Inst v1, v2, s, ARMVCCThen, pred, zero_reg))>;
5652  }
5653}
5654
5655defm MVE_VMLA_qr_s8   : MVE_VMLA_qr_multi<"vmla", MVE_v16s8, 0b0>;
5656defm MVE_VMLA_qr_s16  : MVE_VMLA_qr_multi<"vmla", MVE_v8s16, 0b0>;
5657defm MVE_VMLA_qr_s32  : MVE_VMLA_qr_multi<"vmla", MVE_v4s32, 0b0>;
5658defm MVE_VMLA_qr_u8   : MVE_VMLA_qr_multi<"vmla", MVE_v16u8, 0b0>;
5659defm MVE_VMLA_qr_u16  : MVE_VMLA_qr_multi<"vmla", MVE_v8u16, 0b0>;
5660defm MVE_VMLA_qr_u32  : MVE_VMLA_qr_multi<"vmla", MVE_v4u32, 0b0>;
5661
5662defm MVE_VMLAS_qr_s8  : MVE_VMLA_qr_multi<"vmlas", MVE_v16s8, 0b1>;
5663defm MVE_VMLAS_qr_s16 : MVE_VMLA_qr_multi<"vmlas", MVE_v8s16, 0b1>;
5664defm MVE_VMLAS_qr_s32 : MVE_VMLA_qr_multi<"vmlas", MVE_v4s32, 0b1>;
5665defm MVE_VMLAS_qr_u8  : MVE_VMLA_qr_multi<"vmlas", MVE_v16u8, 0b1>;
5666defm MVE_VMLAS_qr_u16 : MVE_VMLA_qr_multi<"vmlas", MVE_v8u16, 0b1>;
5667defm MVE_VMLAS_qr_u32 : MVE_VMLA_qr_multi<"vmlas", MVE_v4u32, 0b1>;
5668
5669multiclass MVE_VFMA_qr_multi<string iname, MVEVectorVTInfo VTI,
5670                             bit scalar_addend> {
5671  def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, scalar_addend, VTI.Size>;
5672  defvar Inst = !cast<Instruction>(NAME);
5673  defvar pred_int = int_arm_mve_fma_predicated;
5674  defvar v1   = (VTI.Vec MQPR:$v1);
5675  defvar v2   = (VTI.Vec MQPR:$v2);
5676  defvar vs   = (VTI.Vec (ARMvdup (i32 rGPR:$s)));
5677  defvar is   = (i32 rGPR:$s);
5678  defvar pred = (VTI.Pred VCCR:$pred);
5679
5680  let Predicates = [HasMVEFloat] in {
5681    if scalar_addend then {
5682      def : Pat<(VTI.Vec (fma v1, v2, vs)),
5683                (VTI.Vec (Inst v1, v2, is))>;
5684      def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5685                                  (VTI.Vec (fma v1, v2, vs)),
5686                                  v1)),
5687                (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;
5688      def : Pat<(VTI.Vec (pred_int v1, v2, vs, pred)),
5689                (VTI.Vec (Inst v1, v2, is, ARMVCCThen, pred, zero_reg))>;
5690    } else {
5691      def : Pat<(VTI.Vec (fma v1, vs, v2)),
5692                (VTI.Vec (Inst v2, v1, is))>;
5693      def : Pat<(VTI.Vec (fma vs, v1, v2)),
5694                (VTI.Vec (Inst v2, v1, is))>;
5695      def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5696                                  (VTI.Vec (fma vs, v2, v1)),
5697                                  v1)),
5698                (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;
5699      def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5700                                  (VTI.Vec (fma v2, vs, v1)),
5701                                  v1)),
5702                (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;
5703      def : Pat<(VTI.Vec (pred_int v1, vs, v2, pred)),
5704                (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred, zero_reg))>;
5705      def : Pat<(VTI.Vec (pred_int vs, v1, v2, pred)),
5706                (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred, zero_reg))>;
5707    }
5708  }
5709}
5710
5711let Predicates = [HasMVEFloat] in {
5712  defm MVE_VFMA_qr_f16  : MVE_VFMA_qr_multi<"vfma",  MVE_v8f16, 0>;
5713  defm MVE_VFMA_qr_f32  : MVE_VFMA_qr_multi<"vfma",  MVE_v4f32, 0>;
5714  defm MVE_VFMA_qr_Sf16 : MVE_VFMA_qr_multi<"vfmas", MVE_v8f16, 1>;
5715  defm MVE_VFMA_qr_Sf32 : MVE_VFMA_qr_multi<"vfmas", MVE_v4f32, 1>;
5716}
5717
5718class MVE_VQDMLAH_qr<string iname, string suffix, bit U, bits<2> size,
5719                     bit bit_5, bit bit_12, list<dag> pattern=[]>
5720  : MVE_qDestSrc_rSrc<iname, suffix, size, pattern> {
5721
5722  let Inst{28} = U;
5723  let Inst{21-20} = size;
5724  let Inst{16} = 0b0;
5725  let Inst{12} = bit_12;
5726  let Inst{8} = 0b0;
5727  let Inst{5} = bit_5;
5728}
5729
5730multiclass MVE_VQDMLAH_qr_multi<string iname, MVEVectorVTInfo VTI,
5731                                bit bit_5, bit bit_12> {
5732  def "": MVE_VQDMLAH_qr<iname, VTI.Suffix, 0b0, VTI.Size, bit_5, bit_12>;
5733  defvar Inst = !cast<Instruction>(NAME);
5734  defvar unpred_int = !cast<Intrinsic>("int_arm_mve_" # iname);
5735  defvar pred_int = !cast<Intrinsic>("int_arm_mve_" # iname # "_predicated");
5736
5737  let Predicates = [HasMVEInt] in {
5738    def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5739                                   (i32 rGPR:$s))),
5740              (VTI.Vec (Inst       (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5741                                   (i32 rGPR:$s)))>;
5742    def : Pat<(VTI.Vec (pred_int   (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5743                                   (i32 rGPR:$s), (VTI.Pred VCCR:$pred))),
5744              (VTI.Vec (Inst       (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5745                                   (i32 rGPR:$s), ARMVCCThen,
5746                                   (VTI.Pred VCCR:$pred), zero_reg))>;
5747  }
5748}
5749
5750multiclass MVE_VQDMLAH_qr_types<string iname, bit bit_5, bit bit_12> {
5751  defm s8  : MVE_VQDMLAH_qr_multi<iname, MVE_v16s8, bit_5, bit_12>;
5752  defm s16 : MVE_VQDMLAH_qr_multi<iname, MVE_v8s16, bit_5, bit_12>;
5753  defm s32 : MVE_VQDMLAH_qr_multi<iname, MVE_v4s32, bit_5, bit_12>;
5754}
5755
5756defm MVE_VQDMLAH_qr   : MVE_VQDMLAH_qr_types<"vqdmlah",   0b1, 0b0>;
5757defm MVE_VQRDMLAH_qr  : MVE_VQDMLAH_qr_types<"vqrdmlah",  0b0, 0b0>;
5758defm MVE_VQDMLASH_qr  : MVE_VQDMLAH_qr_types<"vqdmlash",  0b1, 0b1>;
5759defm MVE_VQRDMLASH_qr : MVE_VQDMLAH_qr_types<"vqrdmlash", 0b0, 0b1>;
5760
5761class MVE_VxDUP<string iname, string suffix, bits<2> size, bit bit_12,
5762                ValueType VT, SDPatternOperator vxdup>
5763  : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
5764          (ins tGPREven:$Rn_src, MVE_VIDUP_imm:$imm), NoItinerary,
5765          iname, suffix, "$Qd, $Rn, $imm", vpred_r, "$Rn = $Rn_src", size,
5766          [(set (VT MQPR:$Qd), (i32 tGPREven:$Rn),
5767              (vxdup (i32 tGPREven:$Rn_src), (i32 imm:$imm)))]> {
5768  bits<4> Qd;
5769  bits<4> Rn;
5770  bits<2> imm;
5771
5772  let Inst{28} = 0b0;
5773  let Inst{25-23} = 0b100;
5774  let Inst{22} = Qd{3};
5775  let Inst{21-20} = size;
5776  let Inst{19-17} = Rn{3-1};
5777  let Inst{16} = 0b1;
5778  let Inst{15-13} = Qd{2-0};
5779  let Inst{12} = bit_12;
5780  let Inst{11-8} = 0b1111;
5781  let Inst{7} = imm{1};
5782  let Inst{6-1} = 0b110111;
5783  let Inst{0} = imm{0};
5784  let validForTailPredication = 1;
5785  let hasSideEffects = 0;
5786}
5787
5788def MVE_VIDUPu8  : MVE_VxDUP<"vidup", "u8",  0b00, 0b0, v16i8, ARMvidup>;
5789def MVE_VIDUPu16 : MVE_VxDUP<"vidup", "u16", 0b01, 0b0, v8i16, ARMvidup>;
5790def MVE_VIDUPu32 : MVE_VxDUP<"vidup", "u32", 0b10, 0b0, v4i32, ARMvidup>;
5791
5792def MVE_VDDUPu8  : MVE_VxDUP<"vddup", "u8",  0b00, 0b1, v16i8, null_frag>;
5793def MVE_VDDUPu16 : MVE_VxDUP<"vddup", "u16", 0b01, 0b1, v8i16, null_frag>;
5794def MVE_VDDUPu32 : MVE_VxDUP<"vddup", "u32", 0b10, 0b1, v4i32, null_frag>;
5795
5796class MVE_VxWDUP<string iname, string suffix, bits<2> size, bit bit_12,
5797                 list<dag> pattern=[]>
5798  : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
5799          (ins tGPREven:$Rn_src, tGPROdd:$Rm, MVE_VIDUP_imm:$imm), NoItinerary,
5800          iname, suffix, "$Qd, $Rn, $Rm, $imm", vpred_r, "$Rn = $Rn_src", size,
5801          pattern> {
5802  bits<4> Qd;
5803  bits<4> Rm;
5804  bits<4> Rn;
5805  bits<2> imm;
5806
5807  let Inst{28} = 0b0;
5808  let Inst{25-23} = 0b100;
5809  let Inst{22} = Qd{3};
5810  let Inst{21-20} = size;
5811  let Inst{19-17} = Rn{3-1};
5812  let Inst{16} = 0b1;
5813  let Inst{15-13} = Qd{2-0};
5814  let Inst{12} = bit_12;
5815  let Inst{11-8} = 0b1111;
5816  let Inst{7} = imm{1};
5817  let Inst{6-4} = 0b110;
5818  let Inst{3-1} = Rm{3-1};
5819  let Inst{0} = imm{0};
5820  let validForTailPredication = 1;
5821  let hasSideEffects = 0;
5822}
5823
5824def MVE_VIWDUPu8  : MVE_VxWDUP<"viwdup", "u8",  0b00, 0b0>;
5825def MVE_VIWDUPu16 : MVE_VxWDUP<"viwdup", "u16", 0b01, 0b0>;
5826def MVE_VIWDUPu32 : MVE_VxWDUP<"viwdup", "u32", 0b10, 0b0>;
5827
5828def MVE_VDWDUPu8  : MVE_VxWDUP<"vdwdup", "u8",  0b00, 0b1>;
5829def MVE_VDWDUPu16 : MVE_VxWDUP<"vdwdup", "u16", 0b01, 0b1>;
5830def MVE_VDWDUPu32 : MVE_VxWDUP<"vdwdup", "u32", 0b10, 0b1>;
5831
5832let isReMaterializable = 1 in
5833class MVE_VCTPInst<string suffix, bits<2> size, list<dag> pattern=[]>
5834  : MVE_p<(outs VCCR:$P0), (ins rGPR:$Rn), NoItinerary, "vctp", suffix,
5835          "$Rn", vpred_n, "", size, pattern> {
5836  bits<4> Rn;
5837
5838  let Inst{28-27} = 0b10;
5839  let Inst{26-22} = 0b00000;
5840  let Inst{21-20} = size;
5841  let Inst{19-16} = Rn{3-0};
5842  let Inst{15-11} = 0b11101;
5843  let Inst{10-0}  = 0b00000000001;
5844  let Unpredictable{10-0} = 0b11111111111;
5845
5846  let Constraints = "";
5847  let DecoderMethod = "DecodeMveVCTP";
5848  let validForTailPredication = 1;
5849}
5850
5851multiclass MVE_VCTP<MVEVectorVTInfo VTI, Intrinsic intr> {
5852  def "": MVE_VCTPInst<VTI.BitsSuffix, VTI.Size>;
5853  defvar Inst = !cast<Instruction>(NAME);
5854
5855  let Predicates = [HasMVEInt] in {
5856    def : Pat<(intr rGPR:$Rn),
5857              (VTI.Pred (Inst rGPR:$Rn))>;
5858    def : Pat<(and (intr rGPR:$Rn), (VTI.Pred VCCR:$mask)),
5859              (VTI.Pred (Inst rGPR:$Rn, ARMVCCThen, VCCR:$mask, zero_reg))>;
5860  }
5861}
5862
5863defm MVE_VCTP8  : MVE_VCTP<MVE_v16i8, int_arm_mve_vctp8>;
5864defm MVE_VCTP16 : MVE_VCTP<MVE_v8i16, int_arm_mve_vctp16>;
5865defm MVE_VCTP32 : MVE_VCTP<MVE_v4i32, int_arm_mve_vctp32>;
5866defm MVE_VCTP64 : MVE_VCTP<MVE_v2i64, int_arm_mve_vctp64>;
5867
5868// end of mve_qDest_rSrc
5869
5870// start of coproc mov
5871
5872class MVE_VMOV_64bit<dag oops, dag iops, bit to_qreg, string ops, string cstr>
5873  : MVE_VMOV_lane_base<oops, !con(iops, (ins MVEPairVectorIndex2:$idx,
5874                                             MVEPairVectorIndex0:$idx2)),
5875                       NoItinerary, "vmov", "", ops, cstr, []> {
5876  bits<5> Rt;
5877  bits<5> Rt2;
5878  bits<4> Qd;
5879  bit idx;
5880  bit idx2;
5881
5882  let Inst{31-23} = 0b111011000;
5883  let Inst{22} = Qd{3};
5884  let Inst{21} = 0b0;
5885  let Inst{20} = to_qreg;
5886  let Inst{19-16} = Rt2{3-0};
5887  let Inst{15-13} = Qd{2-0};
5888  let Inst{12-5} = 0b01111000;
5889  let Inst{4} = idx2;
5890  let Inst{3-0} = Rt{3-0};
5891
5892  let VecSize = 0b10;
5893  let hasSideEffects = 0;
5894}
5895
5896// The assembly syntax for these instructions mentions the vector
5897// register name twice, e.g.
5898//
5899//    vmov q2[2], q2[0], r0, r1
5900//    vmov r0, r1, q2[2], q2[0]
5901//
5902// which needs a bit of juggling with MC operand handling.
5903//
5904// For the move _into_ a vector register, the MC operand list also has
5905// to mention the register name twice: once as the output, and once as
5906// an extra input to represent where the unchanged half of the output
5907// register comes from (when this instruction is used in code
5908// generation). So we arrange that the first mention of the vector reg
5909// in the instruction is considered by the AsmMatcher to be the output
5910// ($Qd), and the second one is the input ($QdSrc). Binding them
5911// together with the existing 'tie' constraint is enough to enforce at
5912// register allocation time that they have to be the same register.
5913//
5914// For the move _from_ a vector register, there's no way to get round
5915// the fact that both instances of that register name have to be
5916// inputs. They have to be the same register again, but this time, we
5917// can't use a tie constraint, because that has to be between an
5918// output and an input operand. So this time, we have to arrange that
5919// the q-reg appears just once in the MC operand list, in spite of
5920// being mentioned twice in the asm syntax - which needs a custom
5921// AsmMatchConverter.
5922
5923def MVE_VMOV_q_rr : MVE_VMOV_64bit<(outs MQPR:$Qd),
5924                                   (ins MQPR:$QdSrc, rGPR:$Rt, rGPR:$Rt2),
5925                                   0b1, "$Qd$idx, $QdSrc$idx2, $Rt, $Rt2",
5926                                   "$Qd = $QdSrc"> {
5927  let DecoderMethod = "DecodeMVEVMOVDRegtoQ";
5928}
5929
5930def MVE_VMOV_rr_q : MVE_VMOV_64bit<(outs rGPR:$Rt, rGPR:$Rt2), (ins MQPR:$Qd),
5931                                   0b0, "$Rt, $Rt2, $Qd$idx, $Qd$idx2", ""> {
5932  let DecoderMethod = "DecodeMVEVMOVQtoDReg";
5933  let AsmMatchConverter = "cvtMVEVMOVQtoDReg";
5934}
5935
5936let Predicates = [HasMVEInt] in {
5937  // Double lane moves. There are a number of patterns here. We know that the
5938  // insertelt's will be in descending order by index, and need to match the 5
5939  // patterns that might contain 2-0 or 3-1 pairs. These are:
5940  // 3 2 1 0    -> vmovqrr 31; vmovqrr 20
5941  // 3 2 1      -> vmovqrr 31; vmov 2
5942  // 3 1        -> vmovqrr 31
5943  // 2 1 0      -> vmovqrr 20; vmov 1
5944  // 2 0        -> vmovqrr 20
5945  // The other potential patterns will be handled by single lane inserts.
5946  def : Pat<(insertelt (insertelt (insertelt (insertelt (v4i32 MQPR:$src1),
5947                                                        rGPR:$srcA, (i32 0)),
5948                                             rGPR:$srcB, (i32 1)),
5949                                  rGPR:$srcC, (i32 2)),
5950                       rGPR:$srcD, (i32 3)),
5951            (MVE_VMOV_q_rr (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcC, (i32 2), (i32 0)),
5952                           rGPR:$srcB, rGPR:$srcD, (i32 3), (i32 1))>;
5953  def : Pat<(insertelt (insertelt (insertelt (v4i32 MQPR:$src1),
5954                                             rGPR:$srcB, (i32 1)),
5955                                  rGPR:$srcC, (i32 2)),
5956                       rGPR:$srcD, (i32 3)),
5957            (MVE_VMOV_q_rr (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$srcC, (i32 2)),
5958                           rGPR:$srcB, rGPR:$srcD, (i32 3), (i32 1))>;
5959  def : Pat<(insertelt (insertelt (v4i32 MQPR:$src1), rGPR:$srcA, (i32 1)), rGPR:$srcB, (i32 3)),
5960            (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcB, (i32 3), (i32 1))>;
5961  def : Pat<(insertelt (insertelt (insertelt (v4i32 MQPR:$src1),
5962                                             rGPR:$srcB, (i32 0)),
5963                                  rGPR:$srcC, (i32 1)),
5964                       rGPR:$srcD, (i32 2)),
5965            (MVE_VMOV_q_rr (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$srcC, (i32 1)),
5966                           rGPR:$srcB, rGPR:$srcD, (i32 2), (i32 0))>;
5967  def : Pat<(insertelt (insertelt (v4i32 MQPR:$src1), rGPR:$srcA, (i32 0)), rGPR:$srcB, (i32 2)),
5968            (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcB, (i32 2), (i32 0))>;
5969}
5970
5971// end of coproc mov
5972
5973// start of MVE interleaving load/store
5974
5975// Base class for the family of interleaving/deinterleaving
5976// load/stores with names like VLD20.8 and VST43.32.
5977class MVE_vldst24_base<bit writeback, bit fourregs, bits<2> stage, bits<2> size,
5978                       bit load, dag Oops, dag loadIops, dag wbIops,
5979                       string iname, string ops,
5980                       string cstr, list<dag> pattern=[]>
5981  : MVE_MI<Oops, !con(loadIops, wbIops), NoItinerary, iname, ops, cstr, size, pattern> {
5982  bits<4> VQd;
5983  bits<4> Rn;
5984
5985  let Inst{31-22} = 0b1111110010;
5986  let Inst{21} = writeback;
5987  let Inst{20} = load;
5988  let Inst{19-16} = Rn;
5989  let Inst{15-13} = VQd{2-0};
5990  let Inst{12-9} = 0b1111;
5991  let Inst{8-7} = size;
5992  let Inst{6-5} = stage;
5993  let Inst{4-1} = 0b0000;
5994  let Inst{0} = fourregs;
5995
5996  let mayLoad = load;
5997  let mayStore = !eq(load,0);
5998  let hasSideEffects = 0;
5999  let validForTailPredication = load;
6000}
6001
6002// A parameter class used to encapsulate all the ways the writeback
6003// variants of VLD20 and friends differ from the non-writeback ones.
6004class MVE_vldst24_writeback<bit b, dag Oo, dag Io,
6005                            string sy="", string c="", string n=""> {
6006  bit writeback = b;
6007  dag Oops = Oo;
6008  dag Iops = Io;
6009  string syntax = sy;
6010  string cstr = c;
6011  string id_suffix = n;
6012}
6013
6014// Another parameter class that encapsulates the differences between VLD2x
6015// and VLD4x.
6016class MVE_vldst24_nvecs<int n, list<int> s, bit b, RegisterOperand vl> {
6017  int nvecs = n;
6018  list<int> stages = s;
6019  bit bit0 = b;
6020  RegisterOperand VecList = vl;
6021}
6022
6023// A third parameter class that distinguishes VLDnn.8 from .16 from .32.
6024class MVE_vldst24_lanesize<int i, bits<2> b> {
6025  int lanesize = i;
6026  bits<2> sizebits = b;
6027}
6028
6029// A base class for each direction of transfer: one for load, one for
6030// store. I can't make these a fourth independent parametric tuple
6031// class, because they have to take the nvecs tuple class as a
6032// parameter, in order to find the right VecList operand type.
6033
6034class MVE_vld24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,
6035                     MVE_vldst24_writeback wb, string iname,
6036                     list<dag> pattern=[]>
6037  : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 1,
6038                     !con((outs n.VecList:$VQd), wb.Oops),
6039                     (ins n.VecList:$VQdSrc), wb.Iops,
6040                     iname, "$VQd, $Rn" # wb.syntax,
6041                     wb.cstr # ",$VQdSrc = $VQd", pattern>;
6042
6043class MVE_vst24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,
6044                     MVE_vldst24_writeback wb, string iname,
6045                     list<dag> pattern=[]>
6046  : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 0,
6047                     wb.Oops, (ins n.VecList:$VQd), wb.Iops,
6048                     iname, "$VQd, $Rn" # wb.syntax,
6049                     wb.cstr, pattern>;
6050
6051// Actually define all the interleaving loads and stores, by a series
6052// of nested foreaches over number of vectors (VLD2/VLD4); stage
6053// within one of those series (VLDx0/VLDx1/VLDx2/VLDx3); size of
6054// vector lane; writeback or no writeback.
6055foreach n = [MVE_vldst24_nvecs<2, [0,1],     0, VecList2Q>,
6056             MVE_vldst24_nvecs<4, [0,1,2,3], 1, VecList4Q>] in
6057foreach stage = n.stages in
6058foreach s = [MVE_vldst24_lanesize< 8, 0b00>,
6059             MVE_vldst24_lanesize<16, 0b01>,
6060             MVE_vldst24_lanesize<32, 0b10>] in
6061foreach wb = [MVE_vldst24_writeback<
6062                1, (outs rGPR:$wb), (ins t2_nosp_addr_offset_none:$Rn),
6063                "!", "$Rn.base = $wb", "_wb">,
6064              MVE_vldst24_writeback<0, (outs), (ins t2_addr_offset_none:$Rn)>] in {
6065
6066  // For each case within all of those foreaches, define the actual
6067  // instructions. The def names are made by gluing together pieces
6068  // from all the parameter classes, and will end up being things like
6069  // MVE_VLD20_8 and MVE_VST43_16_wb.
6070
6071  def "MVE_VLD" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix
6072    : MVE_vld24_base<n, stage, s.sizebits, wb,
6073                     "vld" # n.nvecs # stage # "." # s.lanesize>;
6074
6075  def "MVE_VST" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix
6076    : MVE_vst24_base<n, stage, s.sizebits, wb,
6077                     "vst" # n.nvecs # stage # "." # s.lanesize>;
6078}
6079
6080def SDTARMVST2    : SDTypeProfile<1, 5, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>,
6081                                         SDTCisSameAs<3, 4>, SDTCisVT<5, i32>]>;
6082def SDTARMVST4    : SDTypeProfile<1, 7, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>, SDTCisVec<3>,
6083                                         SDTCisSameAs<3, 4>, SDTCisSameAs<3, 5>,
6084                                         SDTCisSameAs<3, 6>, SDTCisVT<7, i32>]>;
6085def MVEVST2UPD       : SDNode<"ARMISD::VST2_UPD", SDTARMVST2, [SDNPHasChain, SDNPMemOperand]>;
6086def MVEVST4UPD       : SDNode<"ARMISD::VST4_UPD", SDTARMVST4, [SDNPHasChain, SDNPMemOperand]>;
6087
6088multiclass MVE_vst24_patterns<int lanesize, ValueType VT> {
6089  foreach stage = [0,1] in
6090    def : Pat<(int_arm_mve_vst2q i32:$addr,
6091                (VT MQPR:$v0), (VT MQPR:$v1), (i32 stage)),
6092              (!cast<Instruction>("MVE_VST2"#stage#"_"#lanesize)
6093                (REG_SEQUENCE MQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1),
6094                t2_addr_offset_none:$addr)>;
6095  foreach stage = [0,1] in
6096    def : Pat<(i32 (MVEVST2UPD i32:$addr, (i32 32),
6097                (VT MQPR:$v0), (VT MQPR:$v1), (i32 stage))),
6098              (i32 (!cast<Instruction>("MVE_VST2"#stage#"_"#lanesize#_wb)
6099                (REG_SEQUENCE MQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1),
6100                t2_addr_offset_none:$addr))>;
6101
6102  foreach stage = [0,1,2,3] in
6103    def : Pat<(int_arm_mve_vst4q i32:$addr,
6104                (VT MQPR:$v0), (VT MQPR:$v1),
6105                (VT MQPR:$v2), (VT MQPR:$v3), (i32 stage)),
6106              (!cast<Instruction>("MVE_VST4"#stage#"_"#lanesize)
6107                (REG_SEQUENCE MQQQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1,
6108                                       VT:$v2, qsub_2, VT:$v3, qsub_3),
6109                t2_addr_offset_none:$addr)>;
6110  foreach stage = [0,1,2,3] in
6111    def : Pat<(i32 (MVEVST4UPD i32:$addr, (i32 64),
6112                (VT MQPR:$v0), (VT MQPR:$v1),
6113                (VT MQPR:$v2), (VT MQPR:$v3), (i32 stage))),
6114              (i32 (!cast<Instruction>("MVE_VST4"#stage#"_"#lanesize#_wb)
6115                (REG_SEQUENCE MQQQQPR, VT:$v0, qsub_0, VT:$v1, qsub_1,
6116                                       VT:$v2, qsub_2, VT:$v3, qsub_3),
6117                t2_addr_offset_none:$addr))>;
6118}
6119defm : MVE_vst24_patterns<8, v16i8>;
6120defm : MVE_vst24_patterns<16, v8i16>;
6121defm : MVE_vst24_patterns<32, v4i32>;
6122defm : MVE_vst24_patterns<16, v8f16>;
6123defm : MVE_vst24_patterns<32, v4f32>;
6124
6125// end of MVE interleaving load/store
6126
6127// start of MVE predicable load/store
6128
6129// A parameter class for the direction of transfer.
6130class MVE_ldst_direction<bit b, dag Oo, dag Io, string c=""> {
6131  bit load = b;
6132  dag Oops = Oo;
6133  dag Iops = Io;
6134  string cstr = c;
6135}
6136def MVE_ld: MVE_ldst_direction<1, (outs MQPR:$Qd), (ins), ",@earlyclobber $Qd">;
6137def MVE_st: MVE_ldst_direction<0, (outs), (ins MQPR:$Qd)>;
6138
6139// A parameter class for the size of memory access in a load.
6140class MVE_memsz<bits<2> e, int s, AddrMode m, string mn, list<string> types> {
6141  bits<2> encoding = e;         // opcode bit(s) for encoding
6142  int shift = s;                // shift applied to immediate load offset
6143  AddrMode AM = m;
6144
6145  // For instruction aliases: define the complete list of type
6146  // suffixes at this size, and the canonical ones for loads and
6147  // stores.
6148  string MnemonicLetter = mn;
6149  int TypeBits = !shl(8, s);
6150  string CanonLoadSuffix = ".u" # TypeBits;
6151  string CanonStoreSuffix = "." # TypeBits;
6152  list<string> suffixes = !foreach(letter, types, "." # letter # TypeBits);
6153}
6154
6155// Instances of MVE_memsz.
6156//
6157// (memD doesn't need an AddrMode, because those are only for
6158// contiguous loads, and memD is only used by gather/scatters.)
6159def MVE_memB: MVE_memsz<0b00, 0, AddrModeT2_i7,   "b", ["", "u", "s"]>;
6160def MVE_memH: MVE_memsz<0b01, 1, AddrModeT2_i7s2, "h", ["", "u", "s", "f"]>;
6161def MVE_memW: MVE_memsz<0b10, 2, AddrModeT2_i7s4, "w", ["", "u", "s", "f"]>;
6162def MVE_memD: MVE_memsz<0b11, 3, ?,               "d", ["", "u", "s", "f"]>;
6163
6164// This is the base class for all the MVE loads and stores other than
6165// the interleaving ones. All the non-interleaving loads/stores share
6166// the characteristic that they operate on just one vector register,
6167// so they are VPT-predicable.
6168//
6169// The predication operand is vpred_n, for both loads and stores. For
6170// store instructions, the reason is obvious: if there is no output
6171// register, there can't be a need for an input parameter giving the
6172// output register's previous value. Load instructions also don't need
6173// that input parameter, because unlike MVE data processing
6174// instructions, predicated loads are defined to set the inactive
6175// lanes of the output register to zero, instead of preserving their
6176// input values.
6177class MVE_VLDRSTR_base<MVE_ldst_direction dir, bit U, bit P, bit W, bit opc,
6178                       dag oops, dag iops, string asm, string suffix,
6179                       string ops, string cstr, bits<2> vecsize, list<dag> pattern=[]>
6180 : MVE_p<oops, iops, NoItinerary, asm, suffix, ops, vpred_n, cstr, vecsize, pattern> {
6181  bits<3> Qd;
6182
6183  let Inst{28} = U;
6184  let Inst{25} = 0b0;
6185  let Inst{24} = P;
6186  let Inst{22} = 0b0;
6187  let Inst{21} = W;
6188  let Inst{20} = dir.load;
6189  let Inst{15-13} = Qd{2-0};
6190  let Inst{12} = opc;
6191  let Inst{11-9} = 0b111;
6192
6193  let mayLoad = dir.load;
6194  let mayStore = !eq(dir.load,0);
6195  let hasSideEffects = 0;
6196  let validForTailPredication = 1;
6197}
6198
6199// Contiguous load and store instructions. These come in two main
6200// categories: same-size loads/stores in which 128 bits of vector
6201// register is transferred to or from 128 bits of memory in the most
6202// obvious way, and widening loads / narrowing stores, in which the
6203// size of memory accessed is less than the size of a vector register,
6204// so the load instructions sign- or zero-extend each memory value
6205// into a wider vector lane, and the store instructions truncate
6206// correspondingly.
6207//
6208// The instruction mnemonics for these two classes look reasonably
6209// similar, but the actual encodings are different enough to need two
6210// separate base classes.
6211
6212// Contiguous, same size
6213class MVE_VLDRSTR_cs<MVE_ldst_direction dir, MVE_memsz memsz, bit P, bit W,
6214                     dag oops, dag iops, string asm, string suffix,
6215                     IndexMode im, string ops, string cstr>
6216  : MVE_VLDRSTR_base<dir, 0, P, W, 1, oops, iops, asm, suffix, ops, cstr, memsz.encoding> {
6217  bits<12> addr;
6218  let Inst{23} = addr{7};
6219  let Inst{19-16} = addr{11-8};
6220  let Inst{8-7} = memsz.encoding;
6221  let Inst{6-0} = addr{6-0};
6222
6223  let IM = im;
6224}
6225
6226// Contiguous, widening/narrowing
6227class MVE_VLDRSTR_cw<MVE_ldst_direction dir, MVE_memsz memsz, bit U,
6228                     bit P, bit W, bits<2> size, dag oops, dag iops,
6229                     string asm, string suffix, IndexMode im,
6230                     string ops, string cstr>
6231  : MVE_VLDRSTR_base<dir, U, P, W, 0, oops, iops, asm, suffix, ops, cstr, size> {
6232  bits<11> addr;
6233  let Inst{23} = addr{7};
6234  let Inst{19} = memsz.encoding{0}; // enough to tell 16- from 32-bit
6235  let Inst{18-16} = addr{10-8};
6236  let Inst{8-7} = size;
6237  let Inst{6-0} = addr{6-0};
6238
6239  let IM = im;
6240}
6241
6242// Multiclass wrapper on each of the _cw and _cs base classes, to
6243// generate three writeback modes (none, preindex, postindex).
6244
6245multiclass MVE_VLDRSTR_cw_m<MVE_ldst_direction dir, MVE_memsz memsz,
6246                            string asm, string suffix, bit U, bits<2> size> {
6247  let AM = memsz.AM in {
6248    def "" : MVE_VLDRSTR_cw<
6249        dir, memsz, U, 1, 0, size,
6250        dir.Oops, !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),
6251        asm, suffix, IndexModeNone, "$Qd, $addr", "">;
6252
6253    def _pre : MVE_VLDRSTR_cw<
6254        dir, memsz, U, 1, 1, size,
6255        !con((outs tGPR:$wb), dir.Oops),
6256        !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),
6257        asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {
6258      let DecoderMethod = "DecodeMVE_MEM_1_pre<"#memsz.shift#">";
6259    }
6260
6261    def _post : MVE_VLDRSTR_cw<
6262        dir, memsz, U, 0, 1, size,
6263        !con((outs tGPR:$wb), dir.Oops),
6264        !con(dir.Iops, (ins t_addr_offset_none:$Rn,
6265                            t2am_imm7_offset<memsz.shift>:$addr)),
6266        asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {
6267      bits<4> Rn;
6268      let Inst{18-16} = Rn{2-0};
6269    }
6270  }
6271}
6272
6273multiclass MVE_VLDRSTR_cs_m<MVE_ldst_direction dir, MVE_memsz memsz,
6274                            string asm, string suffix> {
6275  let AM = memsz.AM in {
6276    def "" : MVE_VLDRSTR_cs<
6277        dir, memsz, 1, 0,
6278        dir.Oops, !con(dir.Iops, (ins t2addrmode_imm7<memsz.shift>:$addr)),
6279        asm, suffix, IndexModeNone, "$Qd, $addr", "">;
6280
6281    def _pre : MVE_VLDRSTR_cs<
6282        dir, memsz, 1, 1,
6283        !con((outs rGPR:$wb), dir.Oops),
6284        !con(dir.Iops, (ins t2addrmode_imm7_pre<memsz.shift>:$addr)),
6285        asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {
6286      let DecoderMethod = "DecodeMVE_MEM_2_pre<"#memsz.shift#">";
6287    }
6288
6289    def _post : MVE_VLDRSTR_cs<
6290        dir, memsz, 0, 1,
6291        !con((outs rGPR:$wb), dir.Oops),
6292        !con(dir.Iops, (ins t2_nosp_addr_offset_none:$Rn,
6293                            t2am_imm7_offset<memsz.shift>:$addr)),
6294        asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {
6295      bits<4> Rn;
6296      let Inst{19-16} = Rn{3-0};
6297    }
6298  }
6299}
6300
6301// Now actually declare all the contiguous load/stores, via those
6302// multiclasses. The instruction ids coming out of this are the bare
6303// names shown in the defm, with _pre or _post appended for writeback,
6304// e.g. MVE_VLDRBS16, MVE_VSTRB16_pre, MVE_VSTRHU16_post.
6305
6306defm MVE_VLDRBS16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s16", 0, 0b01>;
6307defm MVE_VLDRBS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s32", 0, 0b10>;
6308defm MVE_VLDRBU16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u16", 1, 0b01>;
6309defm MVE_VLDRBU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u32", 1, 0b10>;
6310defm MVE_VLDRHS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "s32", 0, 0b10>;
6311defm MVE_VLDRHU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "u32", 1, 0b10>;
6312
6313defm MVE_VLDRBU8:  MVE_VLDRSTR_cs_m<MVE_ld, MVE_memB, "vldrb", "u8">;
6314defm MVE_VLDRHU16: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memH, "vldrh", "u16">;
6315defm MVE_VLDRWU32: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memW, "vldrw", "u32">;
6316
6317defm MVE_VSTRB16:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "16",  0, 0b01>;
6318defm MVE_VSTRB32:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "32",  0, 0b10>;
6319defm MVE_VSTRH32:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memH, "vstrh", "32",  0, 0b10>;
6320
6321defm MVE_VSTRBU8 : MVE_VLDRSTR_cs_m<MVE_st, MVE_memB, "vstrb", "8">;
6322defm MVE_VSTRHU16: MVE_VLDRSTR_cs_m<MVE_st, MVE_memH, "vstrh", "16">;
6323defm MVE_VSTRWU32: MVE_VLDRSTR_cs_m<MVE_st, MVE_memW, "vstrw", "32">;
6324
6325// Gather loads / scatter stores whose address operand is of the form
6326// [Rn,Qm], i.e. a single GPR as the common base address, plus a
6327// vector of offset from it. ('Load/store this sequence of elements of
6328// the same array.')
6329//
6330// Like the contiguous family, these loads and stores can widen the
6331// loaded values / truncate the stored ones, or they can just
6332// load/store the same size of memory and vector lane. But unlike the
6333// contiguous family, there's no particular difference in encoding
6334// between those two cases.
6335//
6336// This family also comes with the option to scale the offset values
6337// in Qm by the size of the loaded memory (i.e. to treat them as array
6338// indices), or not to scale them (to treat them as plain byte offsets
6339// in memory, so that perhaps the loaded values are unaligned). The
6340// scaled instructions' address operand in assembly looks like
6341// [Rn,Qm,UXTW #2] or similar.
6342
6343// Base class.
6344class MVE_VLDRSTR_rq<MVE_ldst_direction dir, MVE_memsz memsz, bit U,
6345                     bits<2> size, bit os, string asm, string suffix, int shift>
6346  : MVE_VLDRSTR_base<dir, U, 0b0, 0b0, 0, dir.Oops,
6347                     !con(dir.Iops, (ins mve_addr_rq_shift<shift>:$addr)),
6348                     asm, suffix, "$Qd, $addr", dir.cstr, size> {
6349  bits<7> addr;
6350  let Inst{23} = 0b1;
6351  let Inst{19-16} = addr{6-3};
6352  let Inst{8-7} = size;
6353  let Inst{6} = memsz.encoding{1};
6354  let Inst{5} = 0;
6355  let Inst{4} = memsz.encoding{0};
6356  let Inst{3-1} = addr{2-0};
6357  let Inst{0} = os;
6358}
6359
6360// Multiclass that defines the scaled and unscaled versions of an
6361// instruction, when the memory size is wider than a byte. The scaled
6362// version gets the default name like MVE_VLDRBU16_rq; the unscaled /
6363// potentially unaligned version gets a "_u" suffix, e.g.
6364// MVE_VLDRBU16_rq_u.
6365multiclass MVE_VLDRSTR_rq_w<MVE_ldst_direction dir, MVE_memsz memsz,
6366                            string asm, string suffix, bit U, bits<2> size> {
6367  def _u : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;
6368  def "" : MVE_VLDRSTR_rq<dir, memsz, U, size, 1, asm, suffix, memsz.shift>;
6369}
6370
6371// Subclass of MVE_VLDRSTR_rq with the same API as that multiclass,
6372// for use when the memory size is one byte, so there's no 'scaled'
6373// version of the instruction at all. (This is encoded as if it were
6374// unscaled, but named in the default way with no _u suffix.)
6375class MVE_VLDRSTR_rq_b<MVE_ldst_direction dir, MVE_memsz memsz,
6376                       string asm, string suffix, bit U, bits<2> size>
6377  : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;
6378
6379// Multiclasses wrapping that to add ISel patterns for intrinsics.
6380multiclass MVE_VLDR_rq_w<MVE_memsz memsz, list<MVEVectorVTInfo> VTIs> {
6381  defm "": MVE_VLDRSTR_rq_w<MVE_ld, memsz, "vldr" # memsz.MnemonicLetter,
6382                            VTIs[0].Suffix, VTIs[0].Unsigned, VTIs[0].Size>;
6383  defvar Inst = !cast<Instruction>(NAME);
6384  defvar InstU = !cast<Instruction>(NAME # "_u");
6385
6386  foreach VTI = VTIs in
6387  foreach UnsignedFlag = !if(!eq(VTI.Size, memsz.encoding),
6388                             [0,1], [VTI.Unsigned]) in {
6389    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag)),
6390              (VTI.Vec (InstU GPR:$base, MQPR:$offsets))>;
6391    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag)),
6392              (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6393    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag, (VTI.Pred VCCR:$pred))),
6394              (VTI.Vec (InstU GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;
6395    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag, (VTI.Pred VCCR:$pred))),
6396              (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;
6397  }
6398}
6399multiclass MVE_VLDR_rq_b<list<MVEVectorVTInfo> VTIs> {
6400  def "": MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb",
6401                           VTIs[0].Suffix, VTIs[0].Unsigned, VTIs[0].Size>;
6402  defvar Inst = !cast<Instruction>(NAME);
6403
6404  foreach VTI = VTIs in {
6405    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned)),
6406              (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6407    def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned, (VTI.Pred VCCR:$pred))),
6408              (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;
6409  }
6410}
6411multiclass MVE_VSTR_rq_w<MVE_memsz memsz, list<MVEVectorVTInfo> VTIs> {
6412  defm "": MVE_VLDRSTR_rq_w<MVE_st, memsz, "vstr" # memsz.MnemonicLetter,
6413                            VTIs[0].BitsSuffix, 0, VTIs[0].Size>;
6414  defvar Inst = !cast<Instruction>(NAME);
6415  defvar InstU = !cast<Instruction>(NAME # "_u");
6416
6417  foreach VTI = VTIs in {
6418    def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0),
6419              (InstU MQPR:$data, GPR:$base, MQPR:$offsets)>;
6420    def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift),
6421              (Inst MQPR:$data, GPR:$base, MQPR:$offsets)>;
6422    def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0, (VTI.Pred VCCR:$pred)),
6423              (InstU MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg)>;
6424    def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift, (VTI.Pred VCCR:$pred)),
6425              (Inst MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg)>;
6426  }
6427}
6428multiclass MVE_VSTR_rq_b<list<MVEVectorVTInfo> VTIs> {
6429  def "": MVE_VLDRSTR_rq_b<MVE_st, MVE_memB, "vstrb",
6430                           VTIs[0].BitsSuffix, 0, VTIs[0].Size>;
6431  defvar Inst = !cast<Instruction>(NAME);
6432
6433  foreach VTI = VTIs in {
6434    def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0),
6435              (Inst MQPR:$data, GPR:$base, MQPR:$offsets)>;
6436    def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0, (VTI.Pred VCCR:$pred)),
6437              (Inst MQPR:$data, GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg)>;
6438  }
6439}
6440
6441// Actually define all the loads and stores in this family.
6442
6443defm MVE_VLDRBU8_rq : MVE_VLDR_rq_b<[MVE_v16u8,MVE_v16s8]>;
6444defm MVE_VLDRBU16_rq: MVE_VLDR_rq_b<[MVE_v8u16]>;
6445defm MVE_VLDRBS16_rq: MVE_VLDR_rq_b<[MVE_v8s16]>;
6446defm MVE_VLDRBU32_rq: MVE_VLDR_rq_b<[MVE_v4u32]>;
6447defm MVE_VLDRBS32_rq: MVE_VLDR_rq_b<[MVE_v4s32]>;
6448
6449defm MVE_VLDRHU16_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v8u16,MVE_v8s16,MVE_v8f16]>;
6450defm MVE_VLDRHU32_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v4u32]>;
6451defm MVE_VLDRHS32_rq: MVE_VLDR_rq_w<MVE_memH, [MVE_v4s32]>;
6452defm MVE_VLDRWU32_rq: MVE_VLDR_rq_w<MVE_memW, [MVE_v4u32,MVE_v4s32,MVE_v4f32]>;
6453defm MVE_VLDRDU64_rq: MVE_VLDR_rq_w<MVE_memD, [MVE_v2u64,MVE_v2s64]>;
6454
6455defm MVE_VSTRB8_rq  : MVE_VSTR_rq_b<[MVE_v16i8]>;
6456defm MVE_VSTRB16_rq : MVE_VSTR_rq_b<[MVE_v8i16]>;
6457defm MVE_VSTRB32_rq : MVE_VSTR_rq_b<[MVE_v4i32]>;
6458
6459defm MVE_VSTRH16_rq : MVE_VSTR_rq_w<MVE_memH, [MVE_v8i16,MVE_v8f16]>;
6460defm MVE_VSTRH32_rq : MVE_VSTR_rq_w<MVE_memH, [MVE_v4i32]>;
6461defm MVE_VSTRW32_rq : MVE_VSTR_rq_w<MVE_memW, [MVE_v4i32,MVE_v4f32]>;
6462defm MVE_VSTRD64_rq : MVE_VSTR_rq_w<MVE_memD, [MVE_v2i64]>;
6463
6464// Gather loads / scatter stores whose address operand is of the form
6465// [Qm,#imm], i.e. a vector containing a full base address for each
6466// loaded item, plus an immediate offset applied consistently to all
6467// of them. ('Load/store the same field from this vector of pointers
6468// to a structure type.')
6469//
6470// This family requires the vector lane size to be at least 32 bits
6471// (so there's room for an address in each lane at all). It has no
6472// widening/narrowing variants. But it does support preindex
6473// writeback, in which the address vector is updated to hold the
6474// addresses actually loaded from.
6475
6476// Base class.
6477class MVE_VLDRSTR_qi<MVE_ldst_direction dir, MVE_memsz memsz, bit W, dag wbops,
6478                     string asm, string wbAsm, string suffix, string cstr = "">
6479  : MVE_VLDRSTR_base<dir, 1, 1, W, 1, !con(wbops, dir.Oops),
6480                     !con(dir.Iops, (ins mve_addr_q_shift<memsz.shift>:$addr)),
6481                     asm, suffix, "$Qd, $addr" # wbAsm, cstr # dir.cstr, memsz.encoding> {
6482  bits<11> addr;
6483  let Inst{23} = addr{7};
6484  let Inst{19-17} = addr{10-8};
6485  let Inst{16} = 0;
6486  let Inst{8} = memsz.encoding{0}; // enough to distinguish 32- from 64-bit
6487  let Inst{7} = 0;
6488  let Inst{6-0} = addr{6-0};
6489}
6490
6491// Multiclass that generates the non-writeback and writeback variants.
6492multiclass MVE_VLDRSTR_qi_m<MVE_ldst_direction dir, MVE_memsz memsz,
6493                            string asm, string suffix> {
6494  def ""   : MVE_VLDRSTR_qi<dir, memsz, 0, (outs),          asm, "",  suffix>;
6495  def _pre : MVE_VLDRSTR_qi<dir, memsz, 1, (outs MQPR:$wb), asm, "!", suffix,
6496                            "$addr.base = $wb"> {
6497    let DecoderMethod="DecodeMVE_MEM_3_pre<"#memsz.shift#">";
6498  }
6499}
6500
6501// Multiclasses wrapping that one, adding selection patterns for the
6502// non-writeback loads and all the stores. (The writeback loads must
6503// deliver multiple output values, so they have to be selected by C++
6504// code.)
6505multiclass MVE_VLDR_qi<MVE_memsz memsz, MVEVectorVTInfo AVTI,
6506                       list<MVEVectorVTInfo> DVTIs> {
6507  defm "" : MVE_VLDRSTR_qi_m<MVE_ld, memsz, "vldr" # memsz.MnemonicLetter,
6508                             "u" # memsz.TypeBits>;
6509  defvar Inst = !cast<Instruction>(NAME);
6510
6511  foreach DVTI = DVTIs in {
6512    def : Pat<(DVTI.Vec (int_arm_mve_vldr_gather_base
6513                 (AVTI.Vec MQPR:$addr), (i32 imm:$offset))),
6514              (DVTI.Vec (Inst (AVTI.Vec MQPR:$addr), (i32 imm:$offset)))>;
6515    def : Pat<(DVTI.Vec (int_arm_mve_vldr_gather_base_predicated
6516                 (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (AVTI.Pred VCCR:$pred))),
6517              (DVTI.Vec (Inst (AVTI.Vec MQPR:$addr), (i32 imm:$offset),
6518                        ARMVCCThen, VCCR:$pred, zero_reg))>;
6519  }
6520}
6521multiclass MVE_VSTR_qi<MVE_memsz memsz, MVEVectorVTInfo AVTI,
6522                       list<MVEVectorVTInfo> DVTIs> {
6523  defm "" : MVE_VLDRSTR_qi_m<MVE_st, memsz, "vstr" # memsz.MnemonicLetter,
6524                             !cast<string>(memsz.TypeBits)>;
6525  defvar Inst = !cast<Instruction>(NAME);
6526  defvar InstPre = !cast<Instruction>(NAME # "_pre");
6527
6528  foreach DVTI = DVTIs in {
6529    def : Pat<(int_arm_mve_vstr_scatter_base
6530                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data)),
6531              (Inst (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6532                    (i32 imm:$offset))>;
6533    def : Pat<(int_arm_mve_vstr_scatter_base_predicated
6534                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data), (AVTI.Pred VCCR:$pred)),
6535              (Inst (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6536                    (i32 imm:$offset), ARMVCCThen, VCCR:$pred, zero_reg)>;
6537    def : Pat<(AVTI.Vec (int_arm_mve_vstr_scatter_base_wb
6538                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data))),
6539              (AVTI.Vec (InstPre (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6540                                 (i32 imm:$offset)))>;
6541    def : Pat<(AVTI.Vec (int_arm_mve_vstr_scatter_base_wb_predicated
6542                (AVTI.Vec MQPR:$addr), (i32 imm:$offset), (DVTI.Vec MQPR:$data), (AVTI.Pred VCCR:$pred))),
6543              (AVTI.Vec (InstPre (DVTI.Vec MQPR:$data), (AVTI.Vec MQPR:$addr),
6544                                 (i32 imm:$offset), ARMVCCThen, VCCR:$pred, zero_reg))>;
6545  }
6546}
6547
6548// Actual instruction definitions.
6549defm MVE_VLDRWU32_qi: MVE_VLDR_qi<MVE_memW, MVE_v4i32, [MVE_v4i32,MVE_v4f32]>;
6550defm MVE_VLDRDU64_qi: MVE_VLDR_qi<MVE_memD, MVE_v2i64, [MVE_v2i64,MVE_v2f64]>;
6551defm MVE_VSTRW32_qi:  MVE_VSTR_qi<MVE_memW, MVE_v4i32, [MVE_v4i32,MVE_v4f32]>;
6552defm MVE_VSTRD64_qi:  MVE_VSTR_qi<MVE_memD, MVE_v2i64, [MVE_v2i64,MVE_v2f64]>;
6553
6554// Define aliases for all the instructions where memory size and
6555// vector lane size are the same. These are mnemonic aliases, so they
6556// apply consistently across all of the above families - contiguous
6557// loads, and both the rq and qi types of gather/scatter.
6558//
6559// Rationale: As long as you're loading (for example) 16-bit memory
6560// values into 16-bit vector lanes, you can think of them as signed or
6561// unsigned integers, fp16 or just raw 16-bit blobs and it makes no
6562// difference. So we permit all of vldrh.16, vldrh.u16, vldrh.s16,
6563// vldrh.f16 and treat them all as equivalent to the canonical
6564// spelling (which happens to be .u16 for loads, and just .16 for
6565// stores).
6566
6567foreach vpt_cond = ["", "t", "e"] in
6568foreach memsz = [MVE_memB, MVE_memH, MVE_memW, MVE_memD] in
6569foreach suffix = memsz.suffixes in {
6570  // Define an alias with every suffix in the list, except for the one
6571  // used by the real Instruction record (i.e. the one that all the
6572  // rest are aliases *for*).
6573
6574  if !ne(suffix, memsz.CanonLoadSuffix) then {
6575    def : MnemonicAlias<
6576      "vldr" # memsz.MnemonicLetter # vpt_cond # suffix,
6577      "vldr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonLoadSuffix>;
6578  }
6579
6580  if !ne(suffix, memsz.CanonStoreSuffix) then {
6581    def : MnemonicAlias<
6582      "vstr" # memsz.MnemonicLetter # vpt_cond # suffix,
6583      "vstr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonStoreSuffix>;
6584  }
6585}
6586
6587// end of MVE predicable load/store
6588
6589class MVE_VPT<string suffix, bits<2> size, dag iops, string asm, list<dag> pattern=[]>
6590  : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm, "", size, pattern> {
6591  bits<3> fc;
6592  bits<4> Mk;
6593  bits<3> Qn;
6594
6595  let Inst{31-23} = 0b111111100;
6596  let Inst{22} = Mk{3};
6597  let Inst{21-20} = size;
6598  let Inst{19-17} = Qn{2-0};
6599  let Inst{16} = 0b1;
6600  let Inst{15-13} = Mk{2-0};
6601  let Inst{12} = fc{2};
6602  let Inst{11-8} = 0b1111;
6603  let Inst{7} = fc{0};
6604  let Inst{4} = 0b0;
6605
6606  let Defs = [VPR];
6607  let validForTailPredication=1;
6608}
6609
6610class MVE_VPTt1<string suffix, bits<2> size, dag iops>
6611  : MVE_VPT<suffix, size, iops, "$fc, $Qn, $Qm"> {
6612  bits<4> Qm;
6613  bits<4> Mk;
6614
6615  let Inst{6} = 0b0;
6616  let Inst{5} = Qm{3};
6617  let Inst{3-1} = Qm{2-0};
6618  let Inst{0} = fc{1};
6619}
6620
6621class MVE_VPTt1i<string suffix, bits<2> size>
6622 : MVE_VPTt1<suffix, size,
6623           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_i:$fc)> {
6624  let Inst{12} = 0b0;
6625  let Inst{0} = 0b0;
6626}
6627
6628def MVE_VPTv4i32 : MVE_VPTt1i<"i32", 0b10>;
6629def MVE_VPTv8i16 : MVE_VPTt1i<"i16", 0b01>;
6630def MVE_VPTv16i8 : MVE_VPTt1i<"i8", 0b00>;
6631
6632class MVE_VPTt1u<string suffix, bits<2> size>
6633 : MVE_VPTt1<suffix, size,
6634           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_u:$fc)> {
6635  let Inst{12} = 0b0;
6636  let Inst{0} = 0b1;
6637}
6638
6639def MVE_VPTv4u32 : MVE_VPTt1u<"u32", 0b10>;
6640def MVE_VPTv8u16 : MVE_VPTt1u<"u16", 0b01>;
6641def MVE_VPTv16u8 : MVE_VPTt1u<"u8", 0b00>;
6642
6643class MVE_VPTt1s<string suffix, bits<2> size>
6644 : MVE_VPTt1<suffix, size,
6645           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_s:$fc)> {
6646  let Inst{12} = 0b1;
6647}
6648
6649def MVE_VPTv4s32 : MVE_VPTt1s<"s32", 0b10>;
6650def MVE_VPTv8s16 : MVE_VPTt1s<"s16", 0b01>;
6651def MVE_VPTv16s8 : MVE_VPTt1s<"s8", 0b00>;
6652
6653class MVE_VPTt2<string suffix, bits<2> size, dag iops>
6654  : MVE_VPT<suffix, size, iops,
6655          "$fc, $Qn, $Rm"> {
6656  bits<4> Rm;
6657  bits<3> fc;
6658  bits<4> Mk;
6659
6660  let Inst{6} = 0b1;
6661  let Inst{5} = fc{1};
6662  let Inst{3-0} = Rm{3-0};
6663}
6664
6665class MVE_VPTt2i<string suffix, bits<2> size>
6666  : MVE_VPTt2<suffix, size,
6667            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_i:$fc)> {
6668  let Inst{12} = 0b0;
6669  let Inst{5} = 0b0;
6670}
6671
6672def MVE_VPTv4i32r : MVE_VPTt2i<"i32", 0b10>;
6673def MVE_VPTv8i16r : MVE_VPTt2i<"i16", 0b01>;
6674def MVE_VPTv16i8r : MVE_VPTt2i<"i8", 0b00>;
6675
6676class MVE_VPTt2u<string suffix, bits<2> size>
6677  : MVE_VPTt2<suffix, size,
6678            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_u:$fc)> {
6679  let Inst{12} = 0b0;
6680  let Inst{5} = 0b1;
6681}
6682
6683def MVE_VPTv4u32r : MVE_VPTt2u<"u32", 0b10>;
6684def MVE_VPTv8u16r : MVE_VPTt2u<"u16", 0b01>;
6685def MVE_VPTv16u8r : MVE_VPTt2u<"u8", 0b00>;
6686
6687class MVE_VPTt2s<string suffix, bits<2> size>
6688  : MVE_VPTt2<suffix, size,
6689            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_s:$fc)> {
6690  let Inst{12} = 0b1;
6691}
6692
6693def MVE_VPTv4s32r : MVE_VPTt2s<"s32", 0b10>;
6694def MVE_VPTv8s16r : MVE_VPTt2s<"s16", 0b01>;
6695def MVE_VPTv16s8r : MVE_VPTt2s<"s8", 0b00>;
6696
6697
6698class MVE_VPTf<string suffix, bit size, dag iops, string asm, list<dag> pattern=[]>
6699  : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm,
6700            "", !if(size, 0b01, 0b10), pattern> {
6701  bits<3> fc;
6702  bits<4> Mk;
6703  bits<3> Qn;
6704
6705  let Inst{31-29} = 0b111;
6706  let Inst{28} = size;
6707  let Inst{27-23} = 0b11100;
6708  let Inst{22} = Mk{3};
6709  let Inst{21-20} = 0b11;
6710  let Inst{19-17} = Qn{2-0};
6711  let Inst{16} = 0b1;
6712  let Inst{15-13} = Mk{2-0};
6713  let Inst{12} = fc{2};
6714  let Inst{11-8} = 0b1111;
6715  let Inst{7} = fc{0};
6716  let Inst{4} = 0b0;
6717
6718  let Defs = [VPR];
6719  let Predicates = [HasMVEFloat];
6720  let validForTailPredication=1;
6721}
6722
6723class MVE_VPTft1<string suffix, bit size>
6724  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_fp:$fc),
6725          "$fc, $Qn, $Qm"> {
6726  bits<3> fc;
6727  bits<4> Qm;
6728
6729  let Inst{6} = 0b0;
6730  let Inst{5} = Qm{3};
6731  let Inst{3-1} = Qm{2-0};
6732  let Inst{0} = fc{1};
6733}
6734
6735def MVE_VPTv4f32         : MVE_VPTft1<"f32", 0b0>;
6736def MVE_VPTv8f16         : MVE_VPTft1<"f16", 0b1>;
6737
6738class MVE_VPTft2<string suffix, bit size>
6739  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_fp:$fc),
6740          "$fc, $Qn, $Rm"> {
6741  bits<3> fc;
6742  bits<4> Rm;
6743
6744  let Inst{6} = 0b1;
6745  let Inst{5} = fc{1};
6746  let Inst{3-0} = Rm{3-0};
6747}
6748
6749def MVE_VPTv4f32r        : MVE_VPTft2<"f32", 0b0>;
6750def MVE_VPTv8f16r        : MVE_VPTft2<"f16", 0b1>;
6751
6752def MVE_VPST : MVE_MI<(outs ), (ins vpt_mask:$Mk), NoItinerary,
6753       !strconcat("vpst", "${Mk}"), "", "", 0b00, []> {
6754  bits<4> Mk;
6755
6756  let Inst{31-23} = 0b111111100;
6757  let Inst{22} = Mk{3};
6758  let Inst{21-16} = 0b110001;
6759  let Inst{15-13} = Mk{2-0};
6760  let Inst{12-0} = 0b0111101001101;
6761  let Unpredictable{12} = 0b1;
6762  let Unpredictable{7} = 0b1;
6763  let Unpredictable{5} = 0b1;
6764
6765  let Uses = [VPR];
6766  let validForTailPredication = 1;
6767}
6768
6769def MVE_VPSEL : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
6770                      "vpsel", "", "$Qd, $Qn, $Qm", vpred_n, "", 0b00, []> {
6771  bits<4> Qn;
6772  bits<4> Qd;
6773  bits<4> Qm;
6774
6775  let Inst{28} = 0b1;
6776  let Inst{25-23} = 0b100;
6777  let Inst{22} = Qd{3};
6778  let Inst{21-20} = 0b11;
6779  let Inst{19-17} = Qn{2-0};
6780  let Inst{16} = 0b1;
6781  let Inst{15-13} = Qd{2-0};
6782  let Inst{12-9} = 0b0111;
6783  let Inst{8} = 0b1;
6784  let Inst{7} = Qn{3};
6785  let Inst{6} = 0b0;
6786  let Inst{5} = Qm{3};
6787  let Inst{4} = 0b0;
6788  let Inst{3-1} = Qm{2-0};
6789  let Inst{0} = 0b1;
6790}
6791
6792foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32",
6793                  "i8", "i16", "i32",       "f16", "f32"] in
6794def : MVEInstAlias<"vpsel${vp}." # suffix # "\t$Qd, $Qn, $Qm",
6795                   (MVE_VPSEL MQPR:$Qd, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
6796
6797let Predicates = [HasMVEInt] in {
6798  def : Pat<(v16i8 (vselect (v16i1 VCCR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),
6799            (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6800  def : Pat<(v8i16 (vselect (v8i1 VCCR:$pred), (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),
6801            (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6802  def : Pat<(v4i32 (vselect (v4i1 VCCR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),
6803            (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6804  def : Pat<(v2i64 (vselect (v2i1 VCCR:$pred), (v2i64 MQPR:$v1), (v2i64 MQPR:$v2))),
6805            (v2i64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6806
6807  def : Pat<(v8f16 (vselect (v8i1 VCCR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),
6808            (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6809  def : Pat<(v4f32 (vselect (v4i1 VCCR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),
6810            (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6811  def : Pat<(v2f64 (vselect (v2i1 VCCR:$pred), (v2f64 MQPR:$v1), (v2f64 MQPR:$v2))),
6812            (v2f64 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone, VCCR:$pred, zero_reg))>;
6813
6814  def : Pat<(v16i8 (vselect (v16i8 MQPR:$pred), (v16i8 MQPR:$v1), (v16i8 MQPR:$v2))),
6815            (v16i8 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6816                              (MVE_VCMPi8 (v16i8 MQPR:$pred), (MVE_VMOVimmi8 0), ARMCCne), zero_reg))>;
6817  def : Pat<(v8i16 (vselect (v8i16 MQPR:$pred), (v8i16 MQPR:$v1), (v8i16 MQPR:$v2))),
6818            (v8i16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6819                              (MVE_VCMPi16 (v8i16 MQPR:$pred), (MVE_VMOVimmi16 0), ARMCCne), zero_reg))>;
6820  def : Pat<(v4i32 (vselect (v4i32 MQPR:$pred), (v4i32 MQPR:$v1), (v4i32 MQPR:$v2))),
6821            (v4i32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6822                              (MVE_VCMPi32 (v4i32 MQPR:$pred), (MVE_VMOVimmi32 0), ARMCCne), zero_reg))>;
6823
6824  def : Pat<(v8f16 (vselect (v8i16 MQPR:$pred), (v8f16 MQPR:$v1), (v8f16 MQPR:$v2))),
6825            (v8f16 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6826                              (MVE_VCMPi16 (v8i16 MQPR:$pred), (MVE_VMOVimmi16 0), ARMCCne), zero_reg))>;
6827  def : Pat<(v4f32 (vselect (v4i32 MQPR:$pred), (v4f32 MQPR:$v1), (v4f32 MQPR:$v2))),
6828            (v4f32 (MVE_VPSEL MQPR:$v1, MQPR:$v2, ARMVCCNone,
6829                              (MVE_VCMPi32 (v4i32 MQPR:$pred), (MVE_VMOVimmi32 0), ARMCCne), zero_reg))>;
6830
6831  // Pred <-> Int
6832  def : Pat<(v16i8 (zext  (v16i1 VCCR:$pred))),
6833            (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6834  def : Pat<(v8i16 (zext  (v8i1  VCCR:$pred))),
6835            (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6836  def : Pat<(v4i32 (zext  (v4i1  VCCR:$pred))),
6837            (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6838  def : Pat<(v2i64 (zext  (v2i1  VCCR:$pred))),
6839            (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6840
6841  def : Pat<(v16i8 (sext  (v16i1 VCCR:$pred))),
6842            (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6843  def : Pat<(v8i16 (sext  (v8i1  VCCR:$pred))),
6844            (v8i16 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6845  def : Pat<(v4i32 (sext  (v4i1  VCCR:$pred))),
6846            (v4i32 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6847  def : Pat<(v2i64 (sext  (v2i1  VCCR:$pred))),
6848            (v2i64 (MVE_VPSEL (MVE_VMOVimmi8 255), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6849
6850  def : Pat<(v16i8 (anyext  (v16i1 VCCR:$pred))),
6851            (v16i8 (MVE_VPSEL (MVE_VMOVimmi8 1), (MVE_VMOVimmi8 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6852  def : Pat<(v8i16 (anyext  (v8i1  VCCR:$pred))),
6853            (v8i16 (MVE_VPSEL (MVE_VMOVimmi16 1), (MVE_VMOVimmi16 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6854  def : Pat<(v4i32 (anyext  (v4i1  VCCR:$pred))),
6855            (v4i32 (MVE_VPSEL (MVE_VMOVimmi32 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6856  def : Pat<(v2i64 (anyext  (v2i1  VCCR:$pred))),
6857            (v2i64 (MVE_VPSEL (MVE_VMOVimmi64 1), (MVE_VMOVimmi32 0), ARMVCCNone, VCCR:$pred, zero_reg))>;
6858}
6859
6860let Predicates = [HasMVEFloat] in {
6861  // Pred <-> Float
6862  // 112 is 1.0 in float
6863  def : Pat<(v4f32 (uint_to_fp (v4i1 VCCR:$pred))),
6864            (v4f32 (MVE_VPSEL (v4f32 (MVE_VMOVimmf32 112)), (v4f32 (MVE_VMOVimmi32 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;
6865  // 2620 in 1.0 in half
6866  def : Pat<(v8f16 (uint_to_fp (v8i1 VCCR:$pred))),
6867            (v8f16 (MVE_VPSEL (v8f16 (MVE_VMOVimmi16 2620)), (v8f16 (MVE_VMOVimmi16 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;
6868  // 240 is -1.0 in float
6869  def : Pat<(v4f32 (sint_to_fp (v4i1 VCCR:$pred))),
6870            (v4f32 (MVE_VPSEL (v4f32 (MVE_VMOVimmf32 240)), (v4f32 (MVE_VMOVimmi32 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;
6871  // 2748 is -1.0 in half
6872  def : Pat<(v8f16 (sint_to_fp (v8i1 VCCR:$pred))),
6873            (v8f16 (MVE_VPSEL (v8f16 (MVE_VMOVimmi16 2748)), (v8f16 (MVE_VMOVimmi16 0)), ARMVCCNone, VCCR:$pred, zero_reg))>;
6874
6875  def : Pat<(v4i1 (fp_to_uint (v4f32 MQPR:$v1))),
6876            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, ARMCCne))>;
6877  def : Pat<(v8i1 (fp_to_uint (v8f16 MQPR:$v1))),
6878            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, ARMCCne))>;
6879  def : Pat<(v4i1 (fp_to_sint (v4f32 MQPR:$v1))),
6880            (v4i1 (MVE_VCMPf32r (v4f32 MQPR:$v1), ZR, ARMCCne))>;
6881  def : Pat<(v8i1 (fp_to_sint (v8f16 MQPR:$v1))),
6882            (v8i1 (MVE_VCMPf16r (v8f16 MQPR:$v1), ZR, ARMCCne))>;
6883}
6884
6885def MVE_VPNOT : MVE_p<(outs VCCR:$P0), (ins VCCR:$P0_in), NoItinerary,
6886                      "vpnot", "", "", vpred_n, "", 0b00, []> {
6887  let Inst{31-0} = 0b11111110001100010000111101001101;
6888  let Unpredictable{19-17} = 0b111;
6889  let Unpredictable{12} = 0b1;
6890  let Unpredictable{7} = 0b1;
6891  let Unpredictable{5} = 0b1;
6892
6893  let Constraints = "";
6894  let DecoderMethod = "DecodeMVEVPNOT";
6895}
6896
6897let Predicates = [HasMVEInt] in {
6898  def : Pat<(v2i1 (xor (v2i1 VCCR:$pred), (v2i1 (predicate_cast (i32 65535))))),
6899            (v2i1 (MVE_VPNOT (v2i1 VCCR:$pred)))>;
6900  def : Pat<(v4i1 (xor (v4i1 VCCR:$pred), (v4i1 (predicate_cast (i32 65535))))),
6901            (v4i1 (MVE_VPNOT (v4i1 VCCR:$pred)))>;
6902  def : Pat<(v8i1 (xor (v8i1 VCCR:$pred), (v8i1 (predicate_cast (i32 65535))))),
6903            (v8i1 (MVE_VPNOT (v8i1 VCCR:$pred)))>;
6904  def : Pat<(v16i1 (xor (v16i1 VCCR:$pred), (v16i1 (predicate_cast (i32 65535))))),
6905            (v16i1 (MVE_VPNOT (v16i1 VCCR:$pred)))>;
6906}
6907
6908
6909class MVE_loltp_start<dag iops, string asm, string ops, bits<2> size>
6910  : t2LOL<(outs GPRlr:$LR), iops, asm, ops> {
6911  bits<4> Rn;
6912  let Predicates = [HasMVEInt];
6913  let Inst{22} = 0b0;
6914  let Inst{21-20} = size;
6915  let Inst{19-16} = Rn{3-0};
6916  let Inst{12} = 0b0;
6917}
6918
6919class MVE_DLSTP<string asm, bits<2> size>
6920  : MVE_loltp_start<(ins rGPR:$Rn), asm, "$LR, $Rn", size> {
6921  let Inst{13} = 0b1;
6922  let Inst{11-1} = 0b00000000000;
6923  let Unpredictable{10-1} = 0b1111111111;
6924}
6925
6926class MVE_WLSTP<string asm, bits<2> size>
6927  : MVE_loltp_start<(ins rGPR:$Rn, wlslabel_u11:$label),
6928                    asm, "$LR, $Rn, $label", size> {
6929  bits<11> label;
6930  let Inst{13} = 0b0;
6931  let Inst{11} = label{0};
6932  let Inst{10-1} = label{10-1};
6933  let isBranch = 1;
6934  let isTerminator = 1;
6935}
6936
6937def SDT_MVEMEMCPYLOOPNODE
6938    : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisPtrTy<1>, SDTCisVT<2, i32>]>;
6939def MVE_MEMCPYLOOPNODE : SDNode<"ARMISD::MEMCPYLOOP", SDT_MVEMEMCPYLOOPNODE,
6940                                [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
6941
6942let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Defs = [CPSR] in {
6943  def MVE_MEMCPYLOOPINST : PseudoInst<(outs),
6944        (ins rGPR:$dst, rGPR:$src, rGPR:$sz),
6945        NoItinerary,
6946        [(MVE_MEMCPYLOOPNODE rGPR:$dst, rGPR:$src, rGPR:$sz)]>;
6947}
6948
6949def SDT_MVEMEMSETLOOPNODE
6950    : SDTypeProfile<0, 3, [SDTCisPtrTy<0>, SDTCisVT<1, v16i8>, SDTCisVT<2, i32>]>;
6951def MVE_MEMSETLOOPNODE : SDNode<"ARMISD::MEMSETLOOP", SDT_MVEMEMSETLOOPNODE,
6952                                [SDNPHasChain, SDNPMayStore, SDNPMayLoad]>;
6953
6954let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Defs = [CPSR] in {
6955  def MVE_MEMSETLOOPINST : PseudoInst<(outs),
6956        (ins rGPR:$dst, MQPR:$src, rGPR:$sz),
6957        NoItinerary,
6958        [(MVE_MEMSETLOOPNODE rGPR:$dst, MQPR:$src, rGPR:$sz)]>;
6959}
6960
6961def MVE_DLSTP_8  : MVE_DLSTP<"dlstp.8",  0b00>;
6962def MVE_DLSTP_16 : MVE_DLSTP<"dlstp.16", 0b01>;
6963def MVE_DLSTP_32 : MVE_DLSTP<"dlstp.32", 0b10>;
6964def MVE_DLSTP_64 : MVE_DLSTP<"dlstp.64", 0b11>;
6965
6966def MVE_WLSTP_8  : MVE_WLSTP<"wlstp.8",  0b00>;
6967def MVE_WLSTP_16 : MVE_WLSTP<"wlstp.16", 0b01>;
6968def MVE_WLSTP_32 : MVE_WLSTP<"wlstp.32", 0b10>;
6969def MVE_WLSTP_64 : MVE_WLSTP<"wlstp.64", 0b11>;
6970
6971class MVE_loltp_end<dag oops, dag iops, string asm, string ops>
6972  : t2LOL<oops, iops, asm, ops> {
6973  let Predicates = [HasMVEInt];
6974  let Inst{22-21} = 0b00;
6975  let Inst{19-16} = 0b1111;
6976  let Inst{12} = 0b0;
6977}
6978
6979def MVE_LETP : MVE_loltp_end<(outs GPRlr:$LRout),
6980                             (ins GPRlr:$LRin, lelabel_u11:$label),
6981                             "letp", "$LRin, $label"> {
6982  bits<11> label;
6983  let Inst{20} = 0b1;
6984  let Inst{13} = 0b0;
6985  let Inst{11} = label{0};
6986  let Inst{10-1} = label{10-1};
6987  let isBranch = 1;
6988  let isTerminator = 1;
6989}
6990
6991def MVE_LCTP : MVE_loltp_end<(outs), (ins pred:$p), "lctp${p}", ""> {
6992  let Inst{20} = 0b0;
6993  let Inst{13} = 0b1;
6994  let Inst{11-1} = 0b00000000000;
6995  let Unpredictable{21-20} = 0b11;
6996  let Unpredictable{11-1} = 0b11111111111;
6997}
6998
6999
7000// Pseudo instructions for lowering MQQPR and MQQQQPR stack spills and reloads.
7001// They are equivalent to VLDMDIA/VSTMDIA with a single reg, as opposed to multiple
7002// dreg subregs.
7003
7004let Predicates = [HasMVEInt], AM = AddrMode4 in {
7005let mayStore = 1, hasSideEffects = 0 in {
7006  def MQQPRStore : t2PseudoInst<(outs), (ins MQQPR:$val, GPRnopc:$ptr),
7007                                4, NoItinerary, []>;
7008  def MQQQQPRStore : t2PseudoInst<(outs), (ins MQQQQPR:$val, GPRnopc:$ptr),
7009                                  4, NoItinerary, []>;
7010}
7011let mayLoad = 1, hasSideEffects = 0 in {
7012  def MQQPRLoad : t2PseudoInst<(outs MQQPR:$val), (ins GPRnopc:$ptr),
7013                               4, NoItinerary, []>;
7014  def MQQQQPRLoad : t2PseudoInst<(outs MQQQQPR:$val), (ins GPRnopc:$ptr),
7015                                 4, NoItinerary, []>;
7016}
7017}
7018
7019// Pseudo for lowering MVE Q register COPYs. These will usually get converted
7020// to a "MVE_VORR dst, src, src", but may behave differently in tail predicated
7021// loops to ensure the whole register is copied, not a subset from a
7022// tail-predicated MVE_VORR. In the event we cannot prove a MVE_VORR is valid,
7023// it will become a pair of VMOVD instructions for each half of the Q register.
7024let Predicates = [HasMVEInt], hasSideEffects = 0, isMoveReg = 1,
7025    D = MVEDomain in {
7026  def MQPRCopy : t2PseudoInst<(outs MQPR:$dst), (ins MQPR:$src),
7027                              8, NoItinerary, []>;
7028}
7029
7030
7031//===----------------------------------------------------------------------===//
7032// Patterns
7033//===----------------------------------------------------------------------===//
7034
7035// PatFrags for loads and stores. Often trying to keep semi-consistent names.
7036
7037def aligned32_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
7038                                  (pre_store node:$val, node:$ptr, node:$offset), [{
7039  return cast<StoreSDNode>(N)->getAlignment() >= 4;
7040}]>;
7041def aligned32_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
7042                                   (post_store node:$val, node:$ptr, node:$offset), [{
7043  return cast<StoreSDNode>(N)->getAlignment() >= 4;
7044}]>;
7045def aligned16_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
7046                                  (pre_store node:$val, node:$ptr, node:$offset), [{
7047  return cast<StoreSDNode>(N)->getAlignment() >= 2;
7048}]>;
7049def aligned16_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
7050                                   (post_store node:$val, node:$ptr, node:$offset), [{
7051  return cast<StoreSDNode>(N)->getAlignment() >= 2;
7052}]>;
7053
7054
7055def aligned_maskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7056                                    (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{
7057  auto *Ld = cast<MaskedLoadSDNode>(N);
7058  return Ld->getMemoryVT().getScalarType() == MVT::i8;
7059}]>;
7060def aligned_sextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7061                                        (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{
7062  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
7063}]>;
7064def aligned_zextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7065                                        (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{
7066  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
7067}]>;
7068def aligned_extmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7069                                       (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{
7070  auto *Ld = cast<MaskedLoadSDNode>(N);
7071  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
7072  return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD;
7073}]>;
7074def aligned_maskedloadvi16: PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7075                                    (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{
7076  auto *Ld = cast<MaskedLoadSDNode>(N);
7077  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
7078  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && Ld->getAlignment() >= 2;
7079}]>;
7080def aligned_sextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7081                                         (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{
7082  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
7083}]>;
7084def aligned_zextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7085                                         (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{
7086  return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
7087}]>;
7088def aligned_extmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7089                                        (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{
7090  auto *Ld = cast<MaskedLoadSDNode>(N);
7091  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
7092  return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD;
7093}]>;
7094def aligned_maskedloadvi32: PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
7095                                    (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{
7096  auto *Ld = cast<MaskedLoadSDNode>(N);
7097  EVT ScalarVT = Ld->getMemoryVT().getScalarType();
7098  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && Ld->getAlignment() >= 4;
7099}]>;
7100
7101def aligned_maskedstvi8 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
7102                                  (masked_st node:$val, node:$ptr, undef, node:$pred), [{
7103  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
7104}]>;
7105def aligned_maskedstvi16 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
7106                                   (masked_st node:$val, node:$ptr, undef, node:$pred), [{
7107  auto *St = cast<MaskedStoreSDNode>(N);
7108  EVT ScalarVT = St->getMemoryVT().getScalarType();
7109  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
7110}]>;
7111def aligned_maskedstvi32 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
7112                                   (masked_st node:$val, node:$ptr, undef, node:$pred), [{
7113  auto *St = cast<MaskedStoreSDNode>(N);
7114  EVT ScalarVT = St->getMemoryVT().getScalarType();
7115  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4;
7116}]>;
7117
7118def pre_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask),
7119                              (masked_st node:$val, node:$base, node:$offset, node:$mask), [{
7120  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
7121  return AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
7122}]>;
7123def post_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask),
7124                               (masked_st node:$val, node:$base, node:$offset, node:$mask), [{
7125  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
7126  return AM == ISD::POST_INC || AM == ISD::POST_DEC;
7127}]>;
7128def aligned_pre_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
7129                                         (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
7130  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
7131}]>;
7132def aligned_post_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
7133                                          (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
7134  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
7135}]>;
7136def aligned_pre_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
7137                                          (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
7138  auto *St = cast<MaskedStoreSDNode>(N);
7139  EVT ScalarVT = St->getMemoryVT().getScalarType();
7140  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
7141}]>;
7142def aligned_post_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
7143                                           (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
7144  auto *St = cast<MaskedStoreSDNode>(N);
7145  EVT ScalarVT = St->getMemoryVT().getScalarType();
7146  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
7147}]>;
7148def aligned_pre_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
7149                                          (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
7150  auto *St = cast<MaskedStoreSDNode>(N);
7151  EVT ScalarVT = St->getMemoryVT().getScalarType();
7152  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4;
7153}]>;
7154def aligned_post_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask),
7155                                           (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{
7156  auto *St = cast<MaskedStoreSDNode>(N);
7157  EVT ScalarVT = St->getMemoryVT().getScalarType();
7158  return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4;
7159}]>;
7160
7161
7162// PatFrags for "Aligned" extending / truncating
7163
7164def aligned_extloadvi8  : PatFrag<(ops node:$ptr), (extloadvi8 node:$ptr)>;
7165def aligned_sextloadvi8 : PatFrag<(ops node:$ptr), (sextloadvi8 node:$ptr)>;
7166def aligned_zextloadvi8 : PatFrag<(ops node:$ptr), (zextloadvi8 node:$ptr)>;
7167
7168def aligned_truncstvi8 : PatFrag<(ops node:$val, node:$ptr),
7169                                 (truncstorevi8 node:$val, node:$ptr)>;
7170def aligned_post_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),
7171                                      (post_truncstvi8 node:$val, node:$base, node:$offset)>;
7172def aligned_pre_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset),
7173                                     (pre_truncstvi8 node:$val, node:$base, node:$offset)>;
7174
7175let MinAlignment = 2 in {
7176  def aligned_extloadvi16  : PatFrag<(ops node:$ptr), (extloadvi16 node:$ptr)>;
7177  def aligned_sextloadvi16 : PatFrag<(ops node:$ptr), (sextloadvi16 node:$ptr)>;
7178  def aligned_zextloadvi16 : PatFrag<(ops node:$ptr), (zextloadvi16 node:$ptr)>;
7179
7180  def aligned_truncstvi16 : PatFrag<(ops node:$val, node:$ptr),
7181                                    (truncstorevi16 node:$val, node:$ptr)>;
7182  def aligned_post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
7183                                         (post_truncstvi16 node:$val, node:$base, node:$offset)>;
7184  def aligned_pre_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
7185                                        (pre_truncstvi16 node:$val, node:$base, node:$offset)>;
7186}
7187
7188def truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$pred),
7189                            (masked_st node:$val, node:$base, undef, node:$pred), [{
7190  return cast<MaskedStoreSDNode>(N)->isTruncatingStore();
7191}]>;
7192def aligned_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$pred),
7193                                       (truncmaskedst node:$val, node:$base, node:$pred), [{
7194  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
7195}]>;
7196def aligned_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$pred),
7197                                        (truncmaskedst node:$val, node:$base, node:$pred), [{
7198  auto *St = cast<MaskedStoreSDNode>(N);
7199  EVT ScalarVT = St->getMemoryVT().getScalarType();
7200  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
7201}]>;
7202def pre_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),
7203                                (masked_st node:$val, node:$base, node:$offset, node:$pred), [{
7204  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
7205  return cast<MaskedStoreSDNode>(N)->isTruncatingStore() && (AM == ISD::PRE_INC || AM == ISD::PRE_DEC);
7206}]>;
7207def aligned_pre_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),
7208                                           (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{
7209  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
7210}]>;
7211def aligned_pre_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred),
7212                                            (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{
7213  auto *St = cast<MaskedStoreSDNode>(N);
7214  EVT ScalarVT = St->getMemoryVT().getScalarType();
7215  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
7216}]>;
7217def post_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),
7218                                 (masked_st node:$val, node:$base, node:$offset, node:$postd), [{
7219  ISD::MemIndexedMode AM = cast<MaskedStoreSDNode>(N)->getAddressingMode();
7220  return cast<MaskedStoreSDNode>(N)->isTruncatingStore() && (AM == ISD::POST_INC || AM == ISD::POST_DEC);
7221}]>;
7222def aligned_post_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),
7223                                            (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{
7224  return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
7225}]>;
7226def aligned_post_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd),
7227                                             (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{
7228  auto *St = cast<MaskedStoreSDNode>(N);
7229  EVT ScalarVT = St->getMemoryVT().getScalarType();
7230  return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2;
7231}]>;
7232
7233// Load/store patterns
7234
7235class MVE_vector_store_typed<ValueType Ty, Instruction RegImmInst,
7236                             PatFrag StoreKind, int shift>
7237  : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr),
7238        (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr)>;
7239
7240class MVE_vector_maskedstore_typed<ValueType Ty, Instruction RegImmInst,
7241                                   PatFrag StoreKind, int shift>
7242  : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, VCCR:$pred),
7243        (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;
7244
7245multiclass MVE_vector_store<Instruction RegImmInst, PatFrag StoreKind,
7246                            int shift> {
7247  def : MVE_vector_store_typed<v16i8, RegImmInst, StoreKind, shift>;
7248  def : MVE_vector_store_typed<v8i16, RegImmInst, StoreKind, shift>;
7249  def : MVE_vector_store_typed<v8f16, RegImmInst, StoreKind, shift>;
7250  def : MVE_vector_store_typed<v4i32, RegImmInst, StoreKind, shift>;
7251  def : MVE_vector_store_typed<v4f32, RegImmInst, StoreKind, shift>;
7252  def : MVE_vector_store_typed<v2i64, RegImmInst, StoreKind, shift>;
7253  def : MVE_vector_store_typed<v2f64, RegImmInst, StoreKind, shift>;
7254}
7255
7256class MVE_vector_load_typed<ValueType Ty, Instruction RegImmInst,
7257                            PatFrag LoadKind, int shift>
7258  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr)),
7259        (Ty (RegImmInst t2addrmode_imm7<shift>:$addr))>;
7260
7261class MVE_vector_maskedload_typed<ValueType Ty, Instruction RegImmInst,
7262                                  PatFrag LoadKind, int shift>
7263  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr, VCCR:$pred, (Ty (ARMvmovImm (i32 0))))),
7264        (Ty (RegImmInst t2addrmode_imm7<shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;
7265
7266multiclass MVE_vector_load<Instruction RegImmInst, PatFrag LoadKind,
7267                           int shift> {
7268  def : MVE_vector_load_typed<v16i8, RegImmInst, LoadKind, shift>;
7269  def : MVE_vector_load_typed<v8i16, RegImmInst, LoadKind, shift>;
7270  def : MVE_vector_load_typed<v8f16, RegImmInst, LoadKind, shift>;
7271  def : MVE_vector_load_typed<v4i32, RegImmInst, LoadKind, shift>;
7272  def : MVE_vector_load_typed<v4f32, RegImmInst, LoadKind, shift>;
7273  def : MVE_vector_load_typed<v2i64, RegImmInst, LoadKind, shift>;
7274  def : MVE_vector_load_typed<v2f64, RegImmInst, LoadKind, shift>;
7275}
7276
7277class MVE_vector_offset_store_typed<ValueType Ty, Instruction Opcode,
7278                                    PatFrag StoreKind, int shift>
7279  : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<shift>:$addr),
7280        (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<shift>:$addr)>;
7281
7282class MVE_vector_offset_maskedstore_typed<ValueType Ty, Instruction Opcode,
7283                                          PatFrag StoreKind, int shift>
7284  : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<shift>:$addr, VCCR:$pred),
7285        (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;
7286
7287multiclass MVE_vector_offset_store<Instruction RegImmInst, PatFrag StoreKind,
7288                                   int shift> {
7289  def : MVE_vector_offset_store_typed<v16i8, RegImmInst, StoreKind, shift>;
7290  def : MVE_vector_offset_store_typed<v8i16, RegImmInst, StoreKind, shift>;
7291  def : MVE_vector_offset_store_typed<v8f16, RegImmInst, StoreKind, shift>;
7292  def : MVE_vector_offset_store_typed<v4i32, RegImmInst, StoreKind, shift>;
7293  def : MVE_vector_offset_store_typed<v4f32, RegImmInst, StoreKind, shift>;
7294  def : MVE_vector_offset_store_typed<v2i64, RegImmInst, StoreKind, shift>;
7295  def : MVE_vector_offset_store_typed<v2f64, RegImmInst, StoreKind, shift>;
7296}
7297
7298
7299let Predicates = [HasMVEInt, IsLE] in {
7300  // Stores
7301  defm : MVE_vector_store<MVE_VSTRBU8, byte_alignedstore, 0>;
7302  defm : MVE_vector_store<MVE_VSTRHU16, hword_alignedstore, 1>;
7303  defm : MVE_vector_store<MVE_VSTRWU32, alignedstore32, 2>;
7304
7305  // Loads
7306  defm : MVE_vector_load<MVE_VLDRBU8, byte_alignedload, 0>;
7307  defm : MVE_vector_load<MVE_VLDRHU16, hword_alignedload, 1>;
7308  defm : MVE_vector_load<MVE_VLDRWU32, alignedload32, 2>;
7309
7310  // Pre/post inc stores
7311  defm : MVE_vector_offset_store<MVE_VSTRBU8_pre, pre_store, 0>;
7312  defm : MVE_vector_offset_store<MVE_VSTRBU8_post, post_store, 0>;
7313  defm : MVE_vector_offset_store<MVE_VSTRHU16_pre, aligned16_pre_store, 1>;
7314  defm : MVE_vector_offset_store<MVE_VSTRHU16_post, aligned16_post_store, 1>;
7315  defm : MVE_vector_offset_store<MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
7316  defm : MVE_vector_offset_store<MVE_VSTRWU32_post, aligned32_post_store, 2>;
7317}
7318
7319let Predicates = [HasMVEInt, IsBE] in {
7320  // Aligned Stores
7321  def : MVE_vector_store_typed<v16i8, MVE_VSTRBU8, store, 0>;
7322  def : MVE_vector_store_typed<v8i16, MVE_VSTRHU16, alignedstore16, 1>;
7323  def : MVE_vector_store_typed<v8f16, MVE_VSTRHU16, alignedstore16, 1>;
7324  def : MVE_vector_store_typed<v4i32, MVE_VSTRWU32, alignedstore32, 2>;
7325  def : MVE_vector_store_typed<v4f32, MVE_VSTRWU32, alignedstore32, 2>;
7326
7327  // Aligned Loads
7328  def : MVE_vector_load_typed<v16i8, MVE_VLDRBU8, load, 0>;
7329  def : MVE_vector_load_typed<v8i16, MVE_VLDRHU16, alignedload16, 1>;
7330  def : MVE_vector_load_typed<v8f16, MVE_VLDRHU16, alignedload16, 1>;
7331  def : MVE_vector_load_typed<v4i32, MVE_VLDRWU32, alignedload32, 2>;
7332  def : MVE_vector_load_typed<v4f32, MVE_VLDRWU32, alignedload32, 2>;
7333
7334  // Other unaligned loads/stores need to go though a VREV
7335  def : Pat<(v2f64 (load t2addrmode_imm7<0>:$addr)),
7336            (v2f64 (MVE_VREV64_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7337  def : Pat<(v2i64 (load t2addrmode_imm7<0>:$addr)),
7338            (v2i64 (MVE_VREV64_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7339  def : Pat<(v4i32 (load t2addrmode_imm7<0>:$addr)),
7340            (v4i32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7341  def : Pat<(v4f32 (load t2addrmode_imm7<0>:$addr)),
7342            (v4f32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7343  def : Pat<(v8i16 (load t2addrmode_imm7<0>:$addr)),
7344            (v8i16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7345  def : Pat<(v8f16 (load t2addrmode_imm7<0>:$addr)),
7346            (v8f16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr)))>;
7347  def : Pat<(store (v2f64 MQPR:$val), t2addrmode_imm7<0>:$addr),
7348            (MVE_VSTRBU8 (MVE_VREV64_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7349  def : Pat<(store (v2i64 MQPR:$val), t2addrmode_imm7<0>:$addr),
7350            (MVE_VSTRBU8 (MVE_VREV64_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7351  def : Pat<(store (v4i32 MQPR:$val), t2addrmode_imm7<0>:$addr),
7352            (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7353  def : Pat<(store (v4f32 MQPR:$val), t2addrmode_imm7<0>:$addr),
7354            (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7355  def : Pat<(store (v8i16 MQPR:$val), t2addrmode_imm7<0>:$addr),
7356            (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7357  def : Pat<(store (v8f16 MQPR:$val), t2addrmode_imm7<0>:$addr),
7358            (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr)>;
7359
7360  // Pre/Post inc stores
7361  def : MVE_vector_offset_store_typed<v16i8, MVE_VSTRBU8_pre, pre_store, 0>;
7362  def : MVE_vector_offset_store_typed<v16i8, MVE_VSTRBU8_post, post_store, 0>;
7363  def : MVE_vector_offset_store_typed<v8i16, MVE_VSTRHU16_pre, aligned16_pre_store, 1>;
7364  def : MVE_vector_offset_store_typed<v8i16, MVE_VSTRHU16_post, aligned16_post_store, 1>;
7365  def : MVE_vector_offset_store_typed<v8f16, MVE_VSTRHU16_pre, aligned16_pre_store, 1>;
7366  def : MVE_vector_offset_store_typed<v8f16, MVE_VSTRHU16_post, aligned16_post_store, 1>;
7367  def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
7368  def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_post, aligned32_post_store, 2>;
7369  def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
7370  def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_post, aligned32_post_store, 2>;
7371}
7372
7373let Predicates = [HasMVEInt] in {
7374  // Aligned masked store, shared between LE and BE
7375  def : MVE_vector_maskedstore_typed<v16i8, MVE_VSTRBU8, aligned_maskedstvi8, 0>;
7376  def : MVE_vector_maskedstore_typed<v8i16, MVE_VSTRHU16, aligned_maskedstvi16, 1>;
7377  def : MVE_vector_maskedstore_typed<v8f16, MVE_VSTRHU16, aligned_maskedstvi16, 1>;
7378  def : MVE_vector_maskedstore_typed<v4i32, MVE_VSTRWU32, aligned_maskedstvi32, 2>;
7379  def : MVE_vector_maskedstore_typed<v4f32, MVE_VSTRWU32, aligned_maskedstvi32, 2>;
7380
7381  // Pre/Post inc masked stores
7382  def : MVE_vector_offset_maskedstore_typed<v16i8, MVE_VSTRBU8_pre, aligned_pre_maskedstorevi8, 0>;
7383  def : MVE_vector_offset_maskedstore_typed<v16i8, MVE_VSTRBU8_post, aligned_post_maskedstorevi8, 0>;
7384  def : MVE_vector_offset_maskedstore_typed<v8i16, MVE_VSTRHU16_pre, aligned_pre_maskedstorevi16, 1>;
7385  def : MVE_vector_offset_maskedstore_typed<v8i16, MVE_VSTRHU16_post, aligned_post_maskedstorevi16, 1>;
7386  def : MVE_vector_offset_maskedstore_typed<v8f16, MVE_VSTRHU16_pre, aligned_pre_maskedstorevi16, 1>;
7387  def : MVE_vector_offset_maskedstore_typed<v8f16, MVE_VSTRHU16_post, aligned_post_maskedstorevi16, 1>;
7388  def : MVE_vector_offset_maskedstore_typed<v4i32, MVE_VSTRWU32_pre, aligned_pre_maskedstorevi32, 2>;
7389  def : MVE_vector_offset_maskedstore_typed<v4i32, MVE_VSTRWU32_post, aligned_post_maskedstorevi32, 2>;
7390  def : MVE_vector_offset_maskedstore_typed<v4f32, MVE_VSTRWU32_pre, aligned_pre_maskedstorevi32, 2>;
7391  def : MVE_vector_offset_maskedstore_typed<v4f32, MVE_VSTRWU32_post, aligned_post_maskedstorevi32, 2>;
7392
7393  // Aligned masked loads
7394  def : MVE_vector_maskedload_typed<v16i8, MVE_VLDRBU8, aligned_maskedloadvi8, 0>;
7395  def : MVE_vector_maskedload_typed<v8i16, MVE_VLDRHU16, aligned_maskedloadvi16, 1>;
7396  def : MVE_vector_maskedload_typed<v8f16, MVE_VLDRHU16, aligned_maskedloadvi16, 1>;
7397  def : MVE_vector_maskedload_typed<v4i32, MVE_VLDRWU32, aligned_maskedloadvi32, 2>;
7398  def : MVE_vector_maskedload_typed<v4f32, MVE_VLDRWU32, aligned_maskedloadvi32, 2>;
7399}
7400
7401// Widening/Narrowing Loads/Stores
7402
7403multiclass MVEExtLoadStore<Instruction LoadSInst, Instruction LoadUInst, string StoreInst,
7404                         string Amble, ValueType VT, int Shift> {
7405  // Trunc stores
7406  def : Pat<(!cast<PatFrag>("aligned_truncst"#Amble) (VT MQPR:$val), taddrmode_imm7<Shift>:$addr),
7407            (!cast<Instruction>(StoreInst) MQPR:$val, taddrmode_imm7<Shift>:$addr)>;
7408  def : Pat<(!cast<PatFrag>("aligned_post_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr),
7409            (!cast<Instruction>(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr)>;
7410  def : Pat<(!cast<PatFrag>("aligned_pre_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr),
7411            (!cast<Instruction>(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr)>;
7412
7413  // Masked trunc stores
7414  def : Pat<(!cast<PatFrag>("aligned_truncmaskedst"#Amble) (VT MQPR:$val), taddrmode_imm7<Shift>:$addr, VCCR:$pred),
7415            (!cast<Instruction>(StoreInst) MQPR:$val, taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;
7416  def : Pat<(!cast<PatFrag>("aligned_post_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, VCCR:$pred),
7417            (!cast<Instruction>(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;
7418  def : Pat<(!cast<PatFrag>("aligned_pre_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, VCCR:$pred),
7419            (!cast<Instruction>(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg)>;
7420
7421  // Ext loads
7422  def : Pat<(VT (!cast<PatFrag>("aligned_extload"#Amble) taddrmode_imm7<Shift>:$addr)),
7423            (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;
7424  def : Pat<(VT (!cast<PatFrag>("aligned_sextload"#Amble) taddrmode_imm7<Shift>:$addr)),
7425            (VT (LoadSInst taddrmode_imm7<Shift>:$addr))>;
7426  def : Pat<(VT (!cast<PatFrag>("aligned_zextload"#Amble) taddrmode_imm7<Shift>:$addr)),
7427            (VT (LoadUInst taddrmode_imm7<Shift>:$addr))>;
7428
7429  // Masked ext loads
7430  def : Pat<(VT (!cast<PatFrag>("aligned_extmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),
7431            (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;
7432  def : Pat<(VT (!cast<PatFrag>("aligned_sextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),
7433            (VT (LoadSInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;
7434  def : Pat<(VT (!cast<PatFrag>("aligned_zextmaskedload"#Amble) taddrmode_imm7<Shift>:$addr, VCCR:$pred, (VT (ARMvmovImm (i32 0))))),
7435            (VT (LoadUInst taddrmode_imm7<Shift>:$addr, ARMVCCThen, VCCR:$pred, zero_reg))>;
7436}
7437
7438let Predicates = [HasMVEInt] in {
7439  defm : MVEExtLoadStore<MVE_VLDRBS16, MVE_VLDRBU16, "MVE_VSTRB16", "vi8", v8i16, 0>;
7440  defm : MVEExtLoadStore<MVE_VLDRBS32, MVE_VLDRBU32, "MVE_VSTRB32", "vi8", v4i32, 0>;
7441  defm : MVEExtLoadStore<MVE_VLDRHS32, MVE_VLDRHU32, "MVE_VSTRH32", "vi16", v4i32, 1>;
7442}
7443
7444
7445// Bit convert patterns
7446
7447let Predicates = [HasMVEInt] in {
7448  def : Pat<(v2f64 (bitconvert (v2i64 MQPR:$src))), (v2f64 MQPR:$src)>;
7449  def : Pat<(v2i64 (bitconvert (v2f64 MQPR:$src))), (v2i64 MQPR:$src)>;
7450
7451  def : Pat<(v4i32 (bitconvert (v4f32 MQPR:$src))), (v4i32 MQPR:$src)>;
7452  def : Pat<(v4f32 (bitconvert (v4i32 MQPR:$src))), (v4f32 MQPR:$src)>;
7453
7454  def : Pat<(v8i16 (bitconvert (v8f16 MQPR:$src))), (v8i16  MQPR:$src)>;
7455  def : Pat<(v8f16 (bitconvert (v8i16 MQPR:$src))), (v8f16  MQPR:$src)>;
7456}
7457
7458let Predicates = [IsLE,HasMVEInt] in {
7459  def : Pat<(v2f64 (bitconvert (v4f32 MQPR:$src))), (v2f64 MQPR:$src)>;
7460  def : Pat<(v2f64 (bitconvert (v4i32 MQPR:$src))), (v2f64 MQPR:$src)>;
7461  def : Pat<(v2f64 (bitconvert (v8f16 MQPR:$src))), (v2f64 MQPR:$src)>;
7462  def : Pat<(v2f64 (bitconvert (v8i16 MQPR:$src))), (v2f64 MQPR:$src)>;
7463  def : Pat<(v2f64 (bitconvert (v16i8 MQPR:$src))), (v2f64 MQPR:$src)>;
7464
7465  def : Pat<(v2i64 (bitconvert (v4f32 MQPR:$src))), (v2i64 MQPR:$src)>;
7466  def : Pat<(v2i64 (bitconvert (v4i32 MQPR:$src))), (v2i64 MQPR:$src)>;
7467  def : Pat<(v2i64 (bitconvert (v8f16 MQPR:$src))), (v2i64 MQPR:$src)>;
7468  def : Pat<(v2i64 (bitconvert (v8i16 MQPR:$src))), (v2i64 MQPR:$src)>;
7469  def : Pat<(v2i64 (bitconvert (v16i8 MQPR:$src))), (v2i64 MQPR:$src)>;
7470
7471  def : Pat<(v4f32 (bitconvert (v2f64 MQPR:$src))), (v4f32 MQPR:$src)>;
7472  def : Pat<(v4f32 (bitconvert (v2i64 MQPR:$src))), (v4f32 MQPR:$src)>;
7473  def : Pat<(v4f32 (bitconvert (v8f16 MQPR:$src))), (v4f32 MQPR:$src)>;
7474  def : Pat<(v4f32 (bitconvert (v8i16 MQPR:$src))), (v4f32 MQPR:$src)>;
7475  def : Pat<(v4f32 (bitconvert (v16i8 MQPR:$src))), (v4f32 MQPR:$src)>;
7476
7477  def : Pat<(v4i32 (bitconvert (v2f64 MQPR:$src))), (v4i32 MQPR:$src)>;
7478  def : Pat<(v4i32 (bitconvert (v2i64 MQPR:$src))), (v4i32 MQPR:$src)>;
7479  def : Pat<(v4i32 (bitconvert (v8f16 MQPR:$src))), (v4i32 MQPR:$src)>;
7480  def : Pat<(v4i32 (bitconvert (v8i16 MQPR:$src))), (v4i32 MQPR:$src)>;
7481  def : Pat<(v4i32 (bitconvert (v16i8 MQPR:$src))), (v4i32 MQPR:$src)>;
7482
7483  def : Pat<(v8f16 (bitconvert (v2f64 MQPR:$src))), (v8f16 MQPR:$src)>;
7484  def : Pat<(v8f16 (bitconvert (v2i64 MQPR:$src))), (v8f16 MQPR:$src)>;
7485  def : Pat<(v8f16 (bitconvert (v4f32 MQPR:$src))), (v8f16 MQPR:$src)>;
7486  def : Pat<(v8f16 (bitconvert (v4i32 MQPR:$src))), (v8f16 MQPR:$src)>;
7487  def : Pat<(v8f16 (bitconvert (v16i8 MQPR:$src))), (v8f16 MQPR:$src)>;
7488
7489  def : Pat<(v8i16 (bitconvert (v2f64 MQPR:$src))), (v8i16 MQPR:$src)>;
7490  def : Pat<(v8i16 (bitconvert (v2i64 MQPR:$src))), (v8i16 MQPR:$src)>;
7491  def : Pat<(v8i16 (bitconvert (v4f32 MQPR:$src))), (v8i16 MQPR:$src)>;
7492  def : Pat<(v8i16 (bitconvert (v4i32 MQPR:$src))), (v8i16 MQPR:$src)>;
7493  def : Pat<(v8i16 (bitconvert (v16i8 MQPR:$src))), (v8i16 MQPR:$src)>;
7494
7495  def : Pat<(v16i8 (bitconvert (v2f64 MQPR:$src))), (v16i8 MQPR:$src)>;
7496  def : Pat<(v16i8 (bitconvert (v2i64 MQPR:$src))), (v16i8 MQPR:$src)>;
7497  def : Pat<(v16i8 (bitconvert (v4f32 MQPR:$src))), (v16i8 MQPR:$src)>;
7498  def : Pat<(v16i8 (bitconvert (v4i32 MQPR:$src))), (v16i8 MQPR:$src)>;
7499  def : Pat<(v16i8 (bitconvert (v8f16 MQPR:$src))), (v16i8 MQPR:$src)>;
7500  def : Pat<(v16i8 (bitconvert (v8i16 MQPR:$src))), (v16i8 MQPR:$src)>;
7501}
7502
7503let Predicates = [IsBE,HasMVEInt] in {
7504  def : Pat<(v2f64 (bitconvert (v4f32 MQPR:$src))), (v2f64 (MVE_VREV64_32 MQPR:$src))>;
7505  def : Pat<(v2f64 (bitconvert (v4i32 MQPR:$src))), (v2f64 (MVE_VREV64_32 MQPR:$src))>;
7506  def : Pat<(v2f64 (bitconvert (v8f16 MQPR:$src))), (v2f64 (MVE_VREV64_16 MQPR:$src))>;
7507  def : Pat<(v2f64 (bitconvert (v8i16 MQPR:$src))), (v2f64 (MVE_VREV64_16 MQPR:$src))>;
7508  def : Pat<(v2f64 (bitconvert (v16i8 MQPR:$src))), (v2f64 (MVE_VREV64_8 MQPR:$src))>;
7509
7510  def : Pat<(v2i64 (bitconvert (v4f32 MQPR:$src))), (v2i64 (MVE_VREV64_32 MQPR:$src))>;
7511  def : Pat<(v2i64 (bitconvert (v4i32 MQPR:$src))), (v2i64 (MVE_VREV64_32 MQPR:$src))>;
7512  def : Pat<(v2i64 (bitconvert (v8f16 MQPR:$src))), (v2i64 (MVE_VREV64_16 MQPR:$src))>;
7513  def : Pat<(v2i64 (bitconvert (v8i16 MQPR:$src))), (v2i64 (MVE_VREV64_16 MQPR:$src))>;
7514  def : Pat<(v2i64 (bitconvert (v16i8 MQPR:$src))), (v2i64 (MVE_VREV64_8 MQPR:$src))>;
7515
7516  def : Pat<(v4f32 (bitconvert (v2f64 MQPR:$src))), (v4f32 (MVE_VREV64_32 MQPR:$src))>;
7517  def : Pat<(v4f32 (bitconvert (v2i64 MQPR:$src))), (v4f32 (MVE_VREV64_32 MQPR:$src))>;
7518  def : Pat<(v4f32 (bitconvert (v8f16 MQPR:$src))), (v4f32 (MVE_VREV32_16 MQPR:$src))>;
7519  def : Pat<(v4f32 (bitconvert (v8i16 MQPR:$src))), (v4f32 (MVE_VREV32_16 MQPR:$src))>;
7520  def : Pat<(v4f32 (bitconvert (v16i8 MQPR:$src))), (v4f32 (MVE_VREV32_8 MQPR:$src))>;
7521
7522  def : Pat<(v4i32 (bitconvert (v2f64 MQPR:$src))), (v4i32 (MVE_VREV64_32 MQPR:$src))>;
7523  def : Pat<(v4i32 (bitconvert (v2i64 MQPR:$src))), (v4i32 (MVE_VREV64_32 MQPR:$src))>;
7524  def : Pat<(v4i32 (bitconvert (v8f16 MQPR:$src))), (v4i32 (MVE_VREV32_16 MQPR:$src))>;
7525  def : Pat<(v4i32 (bitconvert (v8i16 MQPR:$src))), (v4i32 (MVE_VREV32_16 MQPR:$src))>;
7526  def : Pat<(v4i32 (bitconvert (v16i8 MQPR:$src))), (v4i32 (MVE_VREV32_8 MQPR:$src))>;
7527
7528  def : Pat<(v8f16 (bitconvert (v2f64 MQPR:$src))), (v8f16 (MVE_VREV64_16 MQPR:$src))>;
7529  def : Pat<(v8f16 (bitconvert (v2i64 MQPR:$src))), (v8f16 (MVE_VREV64_16 MQPR:$src))>;
7530  def : Pat<(v8f16 (bitconvert (v4f32 MQPR:$src))), (v8f16 (MVE_VREV32_16 MQPR:$src))>;
7531  def : Pat<(v8f16 (bitconvert (v4i32 MQPR:$src))), (v8f16 (MVE_VREV32_16 MQPR:$src))>;
7532  def : Pat<(v8f16 (bitconvert (v16i8 MQPR:$src))), (v8f16 (MVE_VREV16_8 MQPR:$src))>;
7533
7534  def : Pat<(v8i16 (bitconvert (v2f64 MQPR:$src))), (v8i16 (MVE_VREV64_16 MQPR:$src))>;
7535  def : Pat<(v8i16 (bitconvert (v2i64 MQPR:$src))), (v8i16 (MVE_VREV64_16 MQPR:$src))>;
7536  def : Pat<(v8i16 (bitconvert (v4f32 MQPR:$src))), (v8i16 (MVE_VREV32_16 MQPR:$src))>;
7537  def : Pat<(v8i16 (bitconvert (v4i32 MQPR:$src))), (v8i16 (MVE_VREV32_16 MQPR:$src))>;
7538  def : Pat<(v8i16 (bitconvert (v16i8 MQPR:$src))), (v8i16 (MVE_VREV16_8 MQPR:$src))>;
7539
7540  def : Pat<(v16i8 (bitconvert (v2f64 MQPR:$src))), (v16i8 (MVE_VREV64_8 MQPR:$src))>;
7541  def : Pat<(v16i8 (bitconvert (v2i64 MQPR:$src))), (v16i8 (MVE_VREV64_8 MQPR:$src))>;
7542  def : Pat<(v16i8 (bitconvert (v4f32 MQPR:$src))), (v16i8 (MVE_VREV32_8 MQPR:$src))>;
7543  def : Pat<(v16i8 (bitconvert (v4i32 MQPR:$src))), (v16i8 (MVE_VREV32_8 MQPR:$src))>;
7544  def : Pat<(v16i8 (bitconvert (v8f16 MQPR:$src))), (v16i8 (MVE_VREV16_8 MQPR:$src))>;
7545  def : Pat<(v16i8 (bitconvert (v8i16 MQPR:$src))), (v16i8 (MVE_VREV16_8 MQPR:$src))>;
7546}
7547