xref: /freebsd/contrib/llvm-project/llvm/lib/Target/ARM/ARMInstrMVE.td (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1//===-- ARMInstrMVE.td - MVE support for ARM ---------------*- tablegen -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file describes the ARM MVE instruction set.
10//
11//===----------------------------------------------------------------------===//
12
13class ExpandImmAsmOp<string shift> : AsmOperandClass {
14  let Name = !strconcat("ExpandImm", shift);
15  let PredicateMethod = !strconcat("isExpImm<", shift, ">");
16  let RenderMethod = "addImmOperands";
17}
18class InvertedExpandImmAsmOp<string shift, string size> : AsmOperandClass {
19  let Name = !strconcat("InvertedExpandImm", shift, "_", size);
20  let PredicateMethod = !strconcat("isInvertedExpImm<", shift, ",", size, ">");
21  let RenderMethod = "addImmOperands";
22}
23
24class ExpandImm<string shift> : Operand<i32> {
25  let ParserMatchClass = ExpandImmAsmOp<shift>;
26  let EncoderMethod = !strconcat("getExpandedImmOpValue<",shift,",false>");
27  let DecoderMethod = !strconcat("DecodeExpandedImmOperand<",shift,">");
28  let PrintMethod = "printExpandedImmOperand";
29}
30class InvertedExpandImm<string shift, string size> : Operand<i32> {
31  let ParserMatchClass = InvertedExpandImmAsmOp<shift, size>;
32  let EncoderMethod = !strconcat("getExpandedImmOpValue<",shift,",true>");
33  let PrintMethod = "printExpandedImmOperand";
34  // No decoder method needed, because this operand type is only used
35  // by aliases (VAND and VORN)
36}
37
38def expzero00 : ExpandImm<"0">;
39def expzero08 : ExpandImm<"8">;
40def expzero16 : ExpandImm<"16">;
41def expzero24 : ExpandImm<"24">;
42
43def expzero00inv16 : InvertedExpandImm<"0", "16">;
44def expzero08inv16 : InvertedExpandImm<"8", "16">;
45
46def expzero00inv32 : InvertedExpandImm<"0", "32">;
47def expzero08inv32 : InvertedExpandImm<"8", "32">;
48def expzero16inv32 : InvertedExpandImm<"16", "32">;
49def expzero24inv32 : InvertedExpandImm<"24", "32">;
50
51// VPT condition mask
52def vpt_mask : Operand<i32> {
53  let PrintMethod = "printVPTMask";
54  let ParserMatchClass = it_mask_asmoperand;
55  let EncoderMethod = "getVPTMaskOpValue";
56  let DecoderMethod = "DecodeVPTMaskOperand";
57}
58
59// VPT/VCMP restricted predicate for sign invariant types
60def pred_restricted_i_asmoperand : AsmOperandClass {
61  let Name = "CondCodeRestrictedI";
62  let RenderMethod = "addITCondCodeOperands";
63  let PredicateMethod = "isITCondCodeRestrictedI";
64  let ParserMethod = "parseITCondCode";
65  let DiagnosticString = "condition code for sign-independent integer "#
66                         "comparison must be EQ or NE";
67}
68
69// VPT/VCMP restricted predicate for signed types
70def pred_restricted_s_asmoperand : AsmOperandClass {
71  let Name = "CondCodeRestrictedS";
72  let RenderMethod = "addITCondCodeOperands";
73  let PredicateMethod = "isITCondCodeRestrictedS";
74  let ParserMethod = "parseITCondCode";
75  let DiagnosticString = "condition code for signed integer "#
76                         "comparison must be EQ, NE, LT, GT, LE or GE";
77}
78
79// VPT/VCMP restricted predicate for unsigned types
80def pred_restricted_u_asmoperand : AsmOperandClass {
81  let Name = "CondCodeRestrictedU";
82  let RenderMethod = "addITCondCodeOperands";
83  let PredicateMethod = "isITCondCodeRestrictedU";
84  let ParserMethod = "parseITCondCode";
85  let DiagnosticString = "condition code for unsigned integer "#
86                         "comparison must be EQ, NE, HS or HI";
87}
88
89// VPT/VCMP restricted predicate for floating point
90def pred_restricted_fp_asmoperand : AsmOperandClass {
91  let Name = "CondCodeRestrictedFP";
92  let RenderMethod = "addITCondCodeOperands";
93  let PredicateMethod = "isITCondCodeRestrictedFP";
94  let ParserMethod = "parseITCondCode";
95  let DiagnosticString = "condition code for floating-point "#
96                         "comparison must be EQ, NE, LT, GT, LE or GE";
97}
98
99class VCMPPredicateOperand : Operand<i32>;
100
101def pred_basic_i : VCMPPredicateOperand {
102  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
103  let ParserMatchClass = pred_restricted_i_asmoperand;
104  let DecoderMethod = "DecodeRestrictedIPredicateOperand";
105  let EncoderMethod = "getRestrictedCondCodeOpValue";
106}
107
108def pred_basic_u : VCMPPredicateOperand {
109  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
110  let ParserMatchClass = pred_restricted_u_asmoperand;
111  let DecoderMethod = "DecodeRestrictedUPredicateOperand";
112  let EncoderMethod = "getRestrictedCondCodeOpValue";
113}
114
115def pred_basic_s : VCMPPredicateOperand {
116  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
117  let ParserMatchClass = pred_restricted_s_asmoperand;
118  let DecoderMethod = "DecodeRestrictedSPredicateOperand";
119  let EncoderMethod = "getRestrictedCondCodeOpValue";
120}
121
122def pred_basic_fp : VCMPPredicateOperand {
123  let PrintMethod = "printMandatoryRestrictedPredicateOperand";
124  let ParserMatchClass = pred_restricted_fp_asmoperand;
125  let DecoderMethod = "DecodeRestrictedFPPredicateOperand";
126  let EncoderMethod = "getRestrictedCondCodeOpValue";
127}
128
129// Register list operands for interleaving load/stores
130def VecList2QAsmOperand : AsmOperandClass {
131  let Name = "VecListTwoMQ";
132  let ParserMethod = "parseVectorList";
133  let RenderMethod = "addMVEVecListOperands";
134  let DiagnosticString = "operand must be a list of two consecutive "#
135                         "q-registers in range [q0,q7]";
136}
137
138def VecList2Q : RegisterOperand<QQPR, "printMVEVectorListTwoQ"> {
139  let ParserMatchClass = VecList2QAsmOperand;
140  let PrintMethod = "printMVEVectorList<2>";
141}
142
143def VecList4QAsmOperand : AsmOperandClass {
144  let Name = "VecListFourMQ";
145  let ParserMethod = "parseVectorList";
146  let RenderMethod = "addMVEVecListOperands";
147  let DiagnosticString = "operand must be a list of four consecutive "#
148                         "q-registers in range [q0,q7]";
149}
150
151def VecList4Q : RegisterOperand<QQQQPR, "printMVEVectorListFourQ"> {
152  let ParserMatchClass = VecList4QAsmOperand;
153  let PrintMethod = "printMVEVectorList<4>";
154}
155
156// taddrmode_imm7  := reg[r0-r7] +/- (imm7 << shift)
157class TMemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
158  let Name = "TMemImm7Shift"#shift#"Offset";
159  let PredicateMethod = "isMemImm7ShiftedOffset<"#shift#",ARM::tGPRRegClassID>";
160  let RenderMethod = "addMemImmOffsetOperands";
161}
162
163class taddrmode_imm7<int shift> : MemOperand {
164  let ParserMatchClass = TMemImm7ShiftOffsetAsmOperand<shift>;
165  // They are printed the same way as the T2 imm8 version
166  let PrintMethod = "printT2AddrModeImm8Operand<false>";
167  // This can also be the same as the T2 version.
168  let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
169  let DecoderMethod = "DecodeTAddrModeImm7<"#shift#">";
170  let MIOperandInfo = (ops tGPR:$base, i32imm:$offsimm);
171}
172
173// t2addrmode_imm7  := reg +/- (imm7)
174class MemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
175  let Name = "MemImm7Shift"#shift#"Offset";
176  let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
177                        ",ARM::GPRnopcRegClassID>";
178  let RenderMethod = "addMemImmOffsetOperands";
179}
180
181def MemImm7Shift0OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<0>;
182def MemImm7Shift1OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<1>;
183def MemImm7Shift2OffsetAsmOperand : MemImm7ShiftOffsetAsmOperand<2>;
184class T2AddrMode_Imm7<int shift> : MemOperand,
185      ComplexPattern<i32, 2, "SelectT2AddrModeImm7<"#shift#">", []> {
186  let EncoderMethod = "getT2AddrModeImmOpValue<7,"#shift#">";
187  let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 0>";
188  let ParserMatchClass =
189    !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetAsmOperand");
190  let MIOperandInfo = (ops GPRnopc:$base, i32imm:$offsimm);
191}
192
193class t2addrmode_imm7<int shift> : T2AddrMode_Imm7<shift> {
194  // They are printed the same way as the imm8 version
195  let PrintMethod = "printT2AddrModeImm8Operand<false>";
196}
197
198class MemImm7ShiftOffsetWBAsmOperand<int shift> : AsmOperandClass {
199  let Name = "MemImm7Shift"#shift#"OffsetWB";
200  let PredicateMethod = "isMemImm7ShiftedOffset<" # shift #
201                        ",ARM::rGPRRegClassID>";
202  let RenderMethod = "addMemImmOffsetOperands";
203}
204
205def MemImm7Shift0OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<0>;
206def MemImm7Shift1OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<1>;
207def MemImm7Shift2OffsetWBAsmOperand : MemImm7ShiftOffsetWBAsmOperand<2>;
208
209class t2addrmode_imm7_pre<int shift> : T2AddrMode_Imm7<shift> {
210  // They are printed the same way as the imm8 version
211  let PrintMethod = "printT2AddrModeImm8Operand<true>";
212  let ParserMatchClass =
213    !cast<AsmOperandClass>("MemImm7Shift"#shift#"OffsetWBAsmOperand");
214  let DecoderMethod = "DecodeT2AddrModeImm7<"#shift#", 1>";
215  let MIOperandInfo = (ops rGPR:$base, i32imm:$offsim);
216}
217
218class t2am_imm7shiftOffsetAsmOperand<int shift>
219  : AsmOperandClass { let Name = "Imm7Shift"#shift; }
220def t2am_imm7shift0OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<0>;
221def t2am_imm7shift1OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<1>;
222def t2am_imm7shift2OffsetAsmOperand : t2am_imm7shiftOffsetAsmOperand<2>;
223
224class t2am_imm7_offset<int shift> : MemOperand {
225  // They are printed the same way as the imm8 version
226  let PrintMethod = "printT2AddrModeImm8OffsetOperand";
227  let ParserMatchClass =
228    !cast<AsmOperandClass>("t2am_imm7shift"#shift#"OffsetAsmOperand");
229  let EncoderMethod = "getT2ScaledImmOpValue<7,"#shift#">";
230  let DecoderMethod = "DecodeT2Imm7<"#shift#">";
231}
232
233// Operands for gather/scatter loads of the form [Rbase, Qoffsets]
234class MemRegRQOffsetAsmOperand<int shift> : AsmOperandClass {
235  let Name = "MemRegRQS"#shift#"Offset";
236  let PredicateMethod = "isMemRegRQOffset<"#shift#">";
237  let RenderMethod = "addMemRegRQOffsetOperands";
238}
239
240def MemRegRQS0OffsetAsmOperand : MemRegRQOffsetAsmOperand<0>;
241def MemRegRQS1OffsetAsmOperand : MemRegRQOffsetAsmOperand<1>;
242def MemRegRQS2OffsetAsmOperand : MemRegRQOffsetAsmOperand<2>;
243def MemRegRQS3OffsetAsmOperand : MemRegRQOffsetAsmOperand<3>;
244
245// mve_addr_rq_shift  := reg + vreg{ << UXTW #shift}
246class mve_addr_rq_shift<int shift> : MemOperand {
247  let EncoderMethod = "getMveAddrModeRQOpValue";
248  let PrintMethod = "printMveAddrModeRQOperand<"#shift#">";
249  let ParserMatchClass =
250    !cast<AsmOperandClass>("MemRegRQS"#shift#"OffsetAsmOperand");
251  let DecoderMethod = "DecodeMveAddrModeRQ";
252  let MIOperandInfo = (ops GPRnopc:$base, MQPR:$offsreg);
253}
254
255class MemRegQOffsetAsmOperand<int shift> : AsmOperandClass {
256  let Name = "MemRegQS"#shift#"Offset";
257  let PredicateMethod = "isMemRegQOffset<"#shift#">";
258  let RenderMethod = "addMemImmOffsetOperands";
259}
260
261def MemRegQS2OffsetAsmOperand : MemRegQOffsetAsmOperand<2>;
262def MemRegQS3OffsetAsmOperand : MemRegQOffsetAsmOperand<3>;
263
264// mve_addr_q_shift  := vreg {+ #imm7s2/4}
265class mve_addr_q_shift<int shift> : MemOperand {
266  let EncoderMethod = "getMveAddrModeQOpValue<"#shift#">";
267  // Can be printed same way as other reg + imm operands
268  let PrintMethod = "printT2AddrModeImm8Operand<false>";
269  let ParserMatchClass =
270    !cast<AsmOperandClass>("MemRegQS"#shift#"OffsetAsmOperand");
271  let DecoderMethod = "DecodeMveAddrModeQ<"#shift#">";
272  let MIOperandInfo = (ops MQPR:$base, i32imm:$imm);
273}
274
275// --------- Start of base classes for the instructions themselves
276
277class MVE_MI<dag oops, dag iops, InstrItinClass itin, string asm,
278             string ops, string cstr, list<dag> pattern>
279  : Thumb2XI<oops, iops, AddrModeNone, 4, itin, !strconcat(asm, "\t", ops), cstr,
280             pattern>,
281    Requires<[HasMVEInt]> {
282  let D = MVEDomain;
283  let DecoderNamespace = "MVE";
284}
285
286// MVE_p is used for most predicated instructions, to add the cluster
287// of input operands that provides the VPT suffix (none, T or E) and
288// the input predicate register.
289class MVE_p<dag oops, dag iops, InstrItinClass itin, string iname,
290            string suffix, string ops, vpred_ops vpred, string cstr,
291            list<dag> pattern=[]>
292  : MVE_MI<oops, !con(iops, (ins vpred:$vp)), itin,
293           // If the instruction has a suffix, like vadd.f32, then the
294           // VPT predication suffix goes before the dot, so the full
295           // name has to be "vadd${vp}.f32".
296           !strconcat(iname, "${vp}",
297                      !if(!eq(suffix, ""), "", !strconcat(".", suffix))),
298           ops, !strconcat(cstr, vpred.vpred_constraint), pattern> {
299  let Inst{31-29} = 0b111;
300  let Inst{27-26} = 0b11;
301}
302
303class MVE_f<dag oops, dag iops, InstrItinClass itin, string iname,
304            string suffix, string ops, vpred_ops vpred, string cstr,
305            list<dag> pattern=[]>
306  : MVE_p<oops, iops, itin, iname, suffix, ops, vpred, cstr, pattern> {
307  let Predicates = [HasMVEFloat];
308}
309
310class MVE_MI_with_pred<dag oops, dag iops, InstrItinClass itin, string asm,
311                       string ops, string cstr, list<dag> pattern>
312  : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm, !strconcat("\t", ops), cstr,
313             pattern>,
314    Requires<[HasV8_1MMainline, HasMVEInt]> {
315  let D = MVEDomain;
316  let DecoderNamespace = "MVE";
317}
318
319class MVE_VMOV_lane_base<dag oops, dag iops, InstrItinClass itin, string asm,
320                         string suffix, string ops, string cstr,
321                         list<dag> pattern>
322  : Thumb2I<oops, iops, AddrModeNone, 4, itin, asm,
323            !if(!eq(suffix, ""), "", "." # suffix) # "\t" # ops,
324            cstr, pattern>,
325    Requires<[HasV8_1MMainline, HasMVEInt]> {
326  let D = MVEDomain;
327  let DecoderNamespace = "MVE";
328}
329
330class MVE_ScalarShift<string iname, dag oops, dag iops, string asm, string cstr,
331            list<dag> pattern=[]>
332  : MVE_MI_with_pred<oops, iops, NoItinerary, iname, asm, cstr, pattern> {
333  let Inst{31-20} = 0b111010100101;
334  let Inst{8} = 0b1;
335
336}
337
338class MVE_ScalarShiftSingleReg<string iname, dag iops, string asm, string cstr,
339                    list<dag> pattern=[]>
340  : MVE_ScalarShift<iname, (outs rGPR:$RdaDest), iops, asm, cstr, pattern> {
341  bits<4> RdaDest;
342
343  let Inst{19-16} = RdaDest{3-0};
344}
345
346class MVE_ScalarShiftSRegImm<string iname, bits<2> op5_4, list<dag> pattern=[]>
347  : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, long_shift:$imm),
348                     "$RdaSrc, $imm", "$RdaDest = $RdaSrc", pattern> {
349  bits<5> imm;
350
351  let Inst{15} = 0b0;
352  let Inst{14-12} = imm{4-2};
353  let Inst{11-8} = 0b1111;
354  let Inst{7-6} = imm{1-0};
355  let Inst{5-4} = op5_4{1-0};
356  let Inst{3-0} = 0b1111;
357}
358
359def MVE_SQSHL : MVE_ScalarShiftSRegImm<"sqshl", 0b11>;
360def MVE_SRSHR : MVE_ScalarShiftSRegImm<"srshr", 0b10>;
361def MVE_UQSHL : MVE_ScalarShiftSRegImm<"uqshl", 0b00>;
362def MVE_URSHR : MVE_ScalarShiftSRegImm<"urshr", 0b01>;
363
364class MVE_ScalarShiftSRegReg<string iname, bits<2> op5_4, list<dag> pattern=[]>
365  : MVE_ScalarShiftSingleReg<iname, (ins rGPR:$RdaSrc, rGPR:$Rm),
366                     "$RdaSrc, $Rm", "$RdaDest = $RdaSrc", pattern> {
367  bits<4> Rm;
368
369  let Inst{15-12} = Rm{3-0};
370  let Inst{11-8} = 0b1111;
371  let Inst{7-6} = 0b00;
372  let Inst{5-4} = op5_4{1-0};
373  let Inst{3-0} = 0b1101;
374}
375
376def MVE_SQRSHR : MVE_ScalarShiftSRegReg<"sqrshr", 0b10>;
377def MVE_UQRSHL : MVE_ScalarShiftSRegReg<"uqrshl", 0b00>;
378
379class MVE_ScalarShiftDoubleReg<string iname, dag iops, string asm,
380                               string cstr, list<dag> pattern=[]>
381  : MVE_ScalarShift<iname, (outs tGPREven:$RdaLo, tGPROdd:$RdaHi),
382                    iops, asm, cstr, pattern> {
383  bits<4> RdaLo;
384  bits<4> RdaHi;
385
386  let Inst{19-17} = RdaLo{3-1};
387  let Inst{11-9} = RdaHi{3-1};
388}
389
390class MVE_ScalarShiftDRegImm<string iname, bits<2> op5_4, bit op16,
391                             list<dag> pattern=[]>
392  : MVE_ScalarShiftDoubleReg<
393      iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, long_shift:$imm),
394      "$RdaLo, $RdaHi, $imm", "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
395      pattern> {
396  bits<5> imm;
397
398  let Inst{16} = op16;
399  let Inst{15} = 0b0;
400  let Inst{14-12} = imm{4-2};
401  let Inst{7-6} = imm{1-0};
402  let Inst{5-4} = op5_4{1-0};
403  let Inst{3-0} = 0b1111;
404}
405
406class MVE_ScalarShiftDRegReg<string iname, bit op5, bit op16,
407                             list<dag> pattern=[]>
408  : MVE_ScalarShiftDoubleReg<
409     iname, (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, rGPR:$Rm),
410     "$RdaLo, $RdaHi, $Rm", "@earlyclobber $RdaHi,@earlyclobber $RdaLo,"
411                            "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
412     pattern> {
413  bits<4> Rm;
414
415  let Inst{16} = op16;
416  let Inst{15-12} = Rm{3-0};
417  let Inst{7-6} = 0b00;
418  let Inst{5} = op5;
419  let Inst{4} = 0b0;
420  let Inst{3-0} = 0b1101;
421
422  // Custom decoder method because of the following overlapping encodings:
423  // ASRL and SQRSHR
424  // LSLL and UQRSHL
425  // SQRSHRL and SQRSHR
426  // UQRSHLL and UQRSHL
427  let DecoderMethod = "DecodeMVEOverlappingLongShift";
428}
429
430def MVE_ASRLr   : MVE_ScalarShiftDRegReg<"asrl",    0b1,  0b0, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
431                                        (ARMasrl tGPREven:$RdaLo_src,
432                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
433def MVE_ASRLi   : MVE_ScalarShiftDRegImm<"asrl",    0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
434                                        (ARMasrl tGPREven:$RdaLo_src,
435                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
436def MVE_LSLLr   : MVE_ScalarShiftDRegReg<"lsll",    0b0,  0b0, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
437                                        (ARMlsll tGPREven:$RdaLo_src,
438                                        tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
439def MVE_LSLLi   : MVE_ScalarShiftDRegImm<"lsll",    0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
440                                        (ARMlsll tGPREven:$RdaLo_src,
441                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
442def MVE_LSRL    : MVE_ScalarShiftDRegImm<"lsrl",    0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
443                                        (ARMlsrl tGPREven:$RdaLo_src,
444                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
445
446def MVE_SQRSHRL : MVE_ScalarShiftDRegReg<"sqrshrl", 0b1,  0b1>;
447def MVE_SQSHLL  : MVE_ScalarShiftDRegImm<"sqshll",  0b11, 0b1>;
448def MVE_SRSHRL  : MVE_ScalarShiftDRegImm<"srshrl",  0b10, 0b1>;
449
450def MVE_UQRSHLL : MVE_ScalarShiftDRegReg<"uqrshll", 0b0,  0b1>;
451def MVE_UQSHLL  : MVE_ScalarShiftDRegImm<"uqshll",  0b00, 0b1>;
452def MVE_URSHRL  : MVE_ScalarShiftDRegImm<"urshrl",  0b01, 0b1>;
453
454// start of mve_rDest instructions
455
456class MVE_rDest<dag oops, dag iops, InstrItinClass itin,
457                string iname, string suffix,
458                string ops, string cstr, list<dag> pattern=[]>
459// Always use vpred_n and not vpred_r: with the output register being
460// a GPR and not a vector register, there can't be any question of
461// what to put in its inactive lanes.
462  : MVE_p<oops, iops, itin, iname, suffix, ops, vpred_n, cstr, pattern> {
463
464  let Inst{25-23} = 0b101;
465  let Inst{11-9} = 0b111;
466  let Inst{4} = 0b0;
467}
468
469class MVE_VABAV<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
470  : MVE_rDest<(outs rGPR:$Rda), (ins rGPR:$Rda_src, MQPR:$Qn, MQPR:$Qm),
471              NoItinerary, "vabav", suffix, "$Rda, $Qn, $Qm", "$Rda = $Rda_src",
472              pattern> {
473  bits<4> Qm;
474  bits<4> Qn;
475  bits<4> Rda;
476
477  let Inst{28} = U;
478  let Inst{22} = 0b0;
479  let Inst{21-20} = size{1-0};
480  let Inst{19-17} = Qn{2-0};
481  let Inst{16} = 0b0;
482  let Inst{15-12} = Rda{3-0};
483  let Inst{8} = 0b1;
484  let Inst{7} = Qn{3};
485  let Inst{6} = 0b0;
486  let Inst{5} = Qm{3};
487  let Inst{3-1} = Qm{2-0};
488  let Inst{0} = 0b1;
489}
490
491def MVE_VABAVs8  : MVE_VABAV<"s8", 0b0, 0b00>;
492def MVE_VABAVs16 : MVE_VABAV<"s16", 0b0, 0b01>;
493def MVE_VABAVs32 : MVE_VABAV<"s32", 0b0, 0b10>;
494def MVE_VABAVu8  : MVE_VABAV<"u8", 0b1, 0b00>;
495def MVE_VABAVu16 : MVE_VABAV<"u16", 0b1, 0b01>;
496def MVE_VABAVu32 : MVE_VABAV<"u32", 0b1, 0b10>;
497
498class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
499              bit A, bit U, bits<2> size, list<dag> pattern=[]>
500  : MVE_rDest<(outs tGPREven:$Rda), iops, NoItinerary,
501              iname, suffix, "$Rda, $Qm", cstr, pattern> {
502  bits<3> Qm;
503  bits<4> Rda;
504
505  let Inst{28} = U;
506  let Inst{22-20} = 0b111;
507  let Inst{19-18} = size{1-0};
508  let Inst{17-16} = 0b01;
509  let Inst{15-13} = Rda{3-1};
510  let Inst{12} = 0b0;
511  let Inst{8-6} = 0b100;
512  let Inst{5} = A;
513  let Inst{3-1} = Qm{2-0};
514  let Inst{0} = 0b0;
515}
516
517multiclass MVE_VADDV_A<string suffix, bit U, bits<2> size,
518                       list<dag> pattern=[]> {
519  def acc    : MVE_VADDV<"vaddva", suffix,
520                         (ins tGPREven:$Rda_src, MQPR:$Qm), "$Rda = $Rda_src",
521                         0b1, U, size, pattern>;
522  def no_acc : MVE_VADDV<"vaddv", suffix,
523                         (ins MQPR:$Qm), "",
524                         0b0, U, size, pattern>;
525}
526
527defm MVE_VADDVs8  : MVE_VADDV_A<"s8",  0b0, 0b00>;
528defm MVE_VADDVs16 : MVE_VADDV_A<"s16", 0b0, 0b01>;
529defm MVE_VADDVs32 : MVE_VADDV_A<"s32", 0b0, 0b10>;
530defm MVE_VADDVu8  : MVE_VADDV_A<"u8",  0b1, 0b00>;
531defm MVE_VADDVu16 : MVE_VADDV_A<"u16", 0b1, 0b01>;
532defm MVE_VADDVu32 : MVE_VADDV_A<"u32", 0b1, 0b10>;
533
534class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,
535               bit A, bit U, list<dag> pattern=[]>
536  : MVE_rDest<(outs tGPREven:$RdaLo, tGPROdd:$RdaHi), iops, NoItinerary, iname,
537              suffix, "$RdaLo, $RdaHi, $Qm", cstr, pattern> {
538  bits<3> Qm;
539  bits<4> RdaLo;
540  bits<4> RdaHi;
541
542  let Inst{28} = U;
543  let Inst{22-20} = RdaHi{3-1};
544  let Inst{19-18} = 0b10;
545  let Inst{17-16} = 0b01;
546  let Inst{15-13} = RdaLo{3-1};
547  let Inst{12} = 0b0;
548  let Inst{8-6} = 0b100;
549  let Inst{5} = A;
550  let Inst{3-1} = Qm{2-0};
551  let Inst{0} = 0b0;
552}
553
554multiclass MVE_VADDLV_A<string suffix, bit U, list<dag> pattern=[]> {
555  def acc    : MVE_VADDLV<"vaddlva", suffix,
556                        (ins tGPREven:$RdaLo_src, tGPROdd:$RdaHi_src, MQPR:$Qm),
557                        "$RdaLo = $RdaLo_src,$RdaHi = $RdaHi_src",
558                        0b1, U, pattern>;
559  def no_acc : MVE_VADDLV<"vaddlv", suffix,
560                        (ins MQPR:$Qm), "",
561                        0b0, U, pattern>;
562}
563
564
565defm MVE_VADDLVs32 : MVE_VADDLV_A<"s32", 0b0>;
566defm MVE_VADDLVu32 : MVE_VADDLV_A<"u32", 0b1>;
567
568class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
569                     bit bit_17, bit bit_7, list<dag> pattern=[]>
570  : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm),
571              NoItinerary, iname, suffix, "$RdaSrc, $Qm",
572              "$RdaDest = $RdaSrc", pattern> {
573  bits<3> Qm;
574  bits<4> RdaDest;
575
576  let Inst{28} = sz;
577  let Inst{22-20} = 0b110;
578  let Inst{19-18} = 0b11;
579  let Inst{17} = bit_17;
580  let Inst{16} = 0b0;
581  let Inst{15-12} = RdaDest{3-0};
582  let Inst{8} = 0b1;
583  let Inst{7} = bit_7;
584  let Inst{6-5} = 0b00;
585  let Inst{3-1} = Qm{2-0};
586  let Inst{0} = 0b0;
587
588  let Predicates = [HasMVEFloat];
589}
590
591multiclass MVE_VMINMAXNMV_fty<string iname, bit bit_7, list<dag> pattern=[]> {
592  def f32 : MVE_VMINMAXNMV<iname, "f32", 0b0, 0b1, bit_7, pattern>;
593  def f16 : MVE_VMINMAXNMV<iname, "f16", 0b1, 0b1, bit_7, pattern>;
594}
595
596defm MVE_VMINNMV : MVE_VMINMAXNMV_fty<"vminnmv", 0b1>;
597defm MVE_VMAXNMV : MVE_VMINMAXNMV_fty<"vmaxnmv", 0b0>;
598
599multiclass MVE_VMINMAXNMAV_fty<string iname, bit bit_7, list<dag> pattern=[]> {
600  def f32 : MVE_VMINMAXNMV<iname, "f32", 0b0, 0b0, bit_7, pattern>;
601  def f16 : MVE_VMINMAXNMV<iname, "f16", 0b1, 0b0, bit_7, pattern>;
602}
603
604defm MVE_VMINNMAV : MVE_VMINMAXNMAV_fty<"vminnmav", 0b1>;
605defm MVE_VMAXNMAV : MVE_VMINMAXNMAV_fty<"vmaxnmav", 0b0>;
606
607class MVE_VMINMAXV<string iname, string suffix, bit U, bits<2> size,
608                 bit bit_17, bit bit_7, list<dag> pattern=[]>
609  : MVE_rDest<(outs rGPR:$RdaDest), (ins rGPR:$RdaSrc, MQPR:$Qm), NoItinerary,
610              iname, suffix, "$RdaSrc, $Qm", "$RdaDest = $RdaSrc", pattern> {
611  bits<3> Qm;
612  bits<4> RdaDest;
613
614  let Inst{28} = U;
615  let Inst{22-20} = 0b110;
616  let Inst{19-18} = size{1-0};
617  let Inst{17} = bit_17;
618  let Inst{16} = 0b0;
619  let Inst{15-12} = RdaDest{3-0};
620  let Inst{8} = 0b1;
621  let Inst{7} = bit_7;
622  let Inst{6-5} = 0b00;
623  let Inst{3-1} = Qm{2-0};
624  let Inst{0} = 0b0;
625}
626
627multiclass MVE_VMINMAXV_ty<string iname, bit bit_7, list<dag> pattern=[]> {
628  def s8  : MVE_VMINMAXV<iname, "s8",  0b0, 0b00, 0b1, bit_7>;
629  def s16 : MVE_VMINMAXV<iname, "s16", 0b0, 0b01, 0b1, bit_7>;
630  def s32 : MVE_VMINMAXV<iname, "s32", 0b0, 0b10, 0b1, bit_7>;
631  def u8  : MVE_VMINMAXV<iname, "u8",  0b1, 0b00, 0b1, bit_7>;
632  def u16 : MVE_VMINMAXV<iname, "u16", 0b1, 0b01, 0b1, bit_7>;
633  def u32 : MVE_VMINMAXV<iname, "u32", 0b1, 0b10, 0b1, bit_7>;
634}
635
636defm MVE_VMINV : MVE_VMINMAXV_ty<"vminv", 0b1>;
637defm MVE_VMAXV : MVE_VMINMAXV_ty<"vmaxv", 0b0>;
638
639multiclass MVE_VMINMAXAV_ty<string iname, bit bit_7, list<dag> pattern=[]> {
640  def s8  : MVE_VMINMAXV<iname, "s8",  0b0, 0b00, 0b0, bit_7>;
641  def s16 : MVE_VMINMAXV<iname, "s16", 0b0, 0b01, 0b0, bit_7>;
642  def s32 : MVE_VMINMAXV<iname, "s32", 0b0, 0b10, 0b0, bit_7>;
643}
644
645defm MVE_VMINAV : MVE_VMINMAXAV_ty<"vminav", 0b1>;
646defm MVE_VMAXAV : MVE_VMINMAXAV_ty<"vmaxav", 0b0>;
647
648class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,
649                   bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
650                   list<dag> pattern=[]>
651  : MVE_rDest<(outs tGPREven:$RdaDest), iops, NoItinerary, iname, suffix,
652              "$RdaDest, $Qn, $Qm", cstr, pattern> {
653  bits<4> RdaDest;
654  bits<3> Qm;
655  bits<3> Qn;
656
657  let Inst{28} = bit_28;
658  let Inst{22-20} = 0b111;
659  let Inst{19-17} = Qn{2-0};
660  let Inst{16} = sz;
661  let Inst{15-13} = RdaDest{3-1};
662  let Inst{12} = X;
663  let Inst{8} = bit_8;
664  let Inst{7-6} = 0b00;
665  let Inst{5} = A;
666  let Inst{3-1} = Qm{2-0};
667  let Inst{0} = bit_0;
668}
669
670multiclass MVE_VMLAMLSDAV_X<string iname, string suffix, dag iops, string cstr,
671                          bit sz, bit bit_28, bit A, bit bit_8, bit bit_0,
672                          list<dag> pattern=[]> {
673  def _noexch : MVE_VMLAMLSDAV<iname, suffix, iops, cstr, sz,
674                            bit_28, A, 0b0, bit_8, bit_0, pattern>;
675  def _exch   : MVE_VMLAMLSDAV<iname # "x", suffix, iops, cstr, sz,
676                            bit_28, A, 0b1, bit_8, bit_0, pattern>;
677}
678
679multiclass MVE_VMLAMLSDAV_XA<string iname, string suffix, bit sz, bit bit_28,
680                           bit bit_8, bit bit_0, list<dag> pattern=[]> {
681  defm _noacc : MVE_VMLAMLSDAV_X<iname, suffix, (ins MQPR:$Qn, MQPR:$Qm), "",
682                              sz, bit_28, 0b0, bit_8, bit_0, pattern>;
683  defm _acc   : MVE_VMLAMLSDAV_X<iname # "a", suffix,
684                             (ins tGPREven:$RdaSrc, MQPR:$Qn, MQPR:$Qm),
685                             "$RdaDest = $RdaSrc",
686                              sz, bit_28, 0b1, bit_8, bit_0, pattern>;
687}
688
689multiclass MVE_VMLADAV_multi<string suffix, bit sz, bit U, bit bit_8,
690                           list<dag> pattern=[]> {
691  defm "" : MVE_VMLAMLSDAV_XA<"vmladav", suffix, sz, U, bit_8, 0b0, pattern>;
692}
693
694defm MVE_VMLADAVs16 : MVE_VMLADAV_multi<"s16", 0b0, 0b0, 0b0>;
695defm MVE_VMLADAVs32 : MVE_VMLADAV_multi<"s32", 0b1, 0b0, 0b0>;
696defm MVE_VMLADAVu16 : MVE_VMLADAV_multi<"u16", 0b0, 0b1, 0b0>;
697defm MVE_VMLADAVu32 : MVE_VMLADAV_multi<"u32", 0b1, 0b1, 0b0>;
698
699defm MVE_VMLADAVs8 : MVE_VMLADAV_multi<"s8", 0b0, 0b0, 0b1>;
700defm MVE_VMLADAVu8 : MVE_VMLADAV_multi<"u8", 0b0, 0b1, 0b1>;
701
702// vmlav aliases vmladav
703foreach acc = ["_acc", "_noacc"] in {
704  foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32"] in {
705    def : MVEInstAlias<!strconcat("vmlav", !if(!eq(acc, "_acc"), "a", ""),
706                       "${vp}.", suffix, "\t$RdaDest, $Qn, $Qm"),
707                       (!cast<Instruction>("MVE_VMLADAV"#suffix#acc#"_noexch")
708                        tGPREven:$RdaDest, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
709  }
710}
711
712multiclass MVE_VMLSDAV_multi<string suffix, bit sz, bit bit_28,
713                           list<dag> pattern=[]> {
714  defm "" : MVE_VMLAMLSDAV_XA<"vmlsdav", suffix, sz, bit_28, 0b0, 0b1, pattern>;
715}
716
717defm MVE_VMLSDAVs8  : MVE_VMLSDAV_multi<"s8", 0, 0b1>;
718defm MVE_VMLSDAVs16 : MVE_VMLSDAV_multi<"s16", 0, 0b0>;
719defm MVE_VMLSDAVs32 : MVE_VMLSDAV_multi<"s32", 1, 0b0>;
720
721// Base class for VMLALDAV and VMLSLDAV, VRMLALDAVH, VRMLSLDAVH
722class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,
723                       bit sz, bit bit_28, bit A, bit X, bit bit_8, bit bit_0,
724                       list<dag> pattern=[]>
725  : MVE_rDest<(outs tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest), iops, NoItinerary,
726              iname, suffix, "$RdaLoDest, $RdaHiDest, $Qn, $Qm", cstr, pattern> {
727  bits<4> RdaLoDest;
728  bits<4> RdaHiDest;
729  bits<3> Qm;
730  bits<3> Qn;
731
732  let Inst{28} = bit_28;
733  let Inst{22-20} = RdaHiDest{3-1};
734  let Inst{19-17} = Qn{2-0};
735  let Inst{16} = sz;
736  let Inst{15-13} = RdaLoDest{3-1};
737  let Inst{12} = X;
738  let Inst{8} = bit_8;
739  let Inst{7-6} = 0b00;
740  let Inst{5} = A;
741  let Inst{3-1} = Qm{2-0};
742  let Inst{0} = bit_0;
743}
744
745multiclass MVE_VMLALDAVBase_X<string iname, string suffix, dag iops,
746                              string cstr, bit sz, bit bit_28, bit A,
747                              bit bit_8, bit bit_0, list<dag> pattern=[]> {
748  def _noexch : MVE_VMLALDAVBase<iname, suffix, iops, cstr, sz,
749                               bit_28, A, 0b0, bit_8, bit_0, pattern>;
750  def _exch   : MVE_VMLALDAVBase<iname # "x", suffix, iops, cstr, sz,
751                               bit_28, A, 0b1, bit_8, bit_0, pattern>;
752}
753
754multiclass MVE_VMLALDAVBase_XA<string iname, string suffix, bit sz, bit bit_28,
755                             bit bit_8, bit bit_0, list<dag> pattern=[]> {
756  defm _noacc : MVE_VMLALDAVBase_X<
757     iname, suffix, (ins MQPR:$Qn, MQPR:$Qm), "",
758     sz, bit_28, 0b0, bit_8, bit_0, pattern>;
759  defm _acc   : MVE_VMLALDAVBase_X<
760     iname # "a", suffix, (ins tGPREven:$RdaLoSrc, tGPROdd:$RdaHiSrc,
761                               MQPR:$Qn, MQPR:$Qm),
762     "$RdaLoDest = $RdaLoSrc,$RdaHiDest = $RdaHiSrc",
763     sz, bit_28, 0b1, bit_8, bit_0, pattern>;
764}
765
766multiclass MVE_VRMLALDAVH_multi<string suffix, bit U, list<dag> pattern=[]> {
767  defm "" : MVE_VMLALDAVBase_XA<
768     "vrmlaldavh", suffix, 0b0, U, 0b1, 0b0, pattern>;
769}
770
771defm MVE_VRMLALDAVHs32 : MVE_VRMLALDAVH_multi<"s32", 0>;
772defm MVE_VRMLALDAVHu32 : MVE_VRMLALDAVH_multi<"u32", 1>;
773
774// vrmlalvh aliases for vrmlaldavh
775def : MVEInstAlias<"vrmlalvh${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
776                  (MVE_VRMLALDAVHs32_noacc_noexch
777                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
778                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
779def : MVEInstAlias<"vrmlalvha${vp}.s32\t$RdaLo, $RdaHi, $Qn, $Qm",
780                  (MVE_VRMLALDAVHs32_acc_noexch
781                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
782                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
783def : MVEInstAlias<"vrmlalvh${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
784                  (MVE_VRMLALDAVHu32_noacc_noexch
785                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
786                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
787def : MVEInstAlias<"vrmlalvha${vp}.u32\t$RdaLo, $RdaHi, $Qn, $Qm",
788                  (MVE_VRMLALDAVHu32_acc_noexch
789                   tGPREven:$RdaLo, tGPROdd:$RdaHi,
790                   MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
791
792multiclass MVE_VMLALDAV_multi<string suffix, bit sz, bit U,
793                              list<dag> pattern=[]> {
794  defm "" : MVE_VMLALDAVBase_XA<"vmlaldav", suffix, sz, U, 0b0, 0b0, pattern>;
795}
796
797defm MVE_VMLALDAVs16 : MVE_VMLALDAV_multi<"s16", 0b0, 0b0>;
798defm MVE_VMLALDAVs32 : MVE_VMLALDAV_multi<"s32", 0b1, 0b0>;
799defm MVE_VMLALDAVu16 : MVE_VMLALDAV_multi<"u16", 0b0, 0b1>;
800defm MVE_VMLALDAVu32 : MVE_VMLALDAV_multi<"u32", 0b1, 0b1>;
801
802// vmlalv aliases vmlaldav
803foreach acc = ["_acc", "_noacc"] in {
804  foreach suffix = ["s16", "s32", "u16", "u32"] in {
805    def : MVEInstAlias<!strconcat("vmlalv", !if(!eq(acc, "_acc"), "a", ""),
806                       "${vp}.", suffix, "\t$RdaLoDest, $RdaHiDest, $Qn, $Qm"),
807                       (!cast<Instruction>("MVE_VMLALDAV"#suffix#acc#"_noexch")
808                       tGPREven:$RdaLoDest, tGPROdd:$RdaHiDest,
809                       MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
810  }
811}
812
813multiclass MVE_VMLSLDAV_multi<string iname, string suffix, bit sz,
814                            bit bit_28, list<dag> pattern=[]> {
815  defm "" : MVE_VMLALDAVBase_XA<iname, suffix, sz, bit_28, 0b0, 0b1, pattern>;
816}
817
818defm MVE_VMLSLDAVs16   : MVE_VMLSLDAV_multi<"vmlsldav", "s16", 0b0, 0b0>;
819defm MVE_VMLSLDAVs32   : MVE_VMLSLDAV_multi<"vmlsldav", "s32", 0b1, 0b0>;
820defm MVE_VRMLSLDAVHs32 : MVE_VMLSLDAV_multi<"vrmlsldavh", "s32", 0b0, 0b1>;
821
822// end of mve_rDest instructions
823
824// start of mve_comp instructions
825
826class MVE_comp<InstrItinClass itin, string iname, string suffix,
827               string cstr, list<dag> pattern=[]>
828  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), itin, iname, suffix,
829           "$Qd, $Qn, $Qm", vpred_r, cstr, pattern> {
830  bits<4> Qd;
831  bits<4> Qn;
832  bits<4> Qm;
833
834  let Inst{22} = Qd{3};
835  let Inst{19-17} = Qn{2-0};
836  let Inst{16} = 0b0;
837  let Inst{15-13} = Qd{2-0};
838  let Inst{12} = 0b0;
839  let Inst{10-9} = 0b11;
840  let Inst{7} = Qn{3};
841  let Inst{5} = Qm{3};
842  let Inst{3-1} = Qm{2-0};
843  let Inst{0} = 0b0;
844}
845
846class MVE_VMINMAXNM<string iname, string suffix, bit sz, bit bit_21,
847                list<dag> pattern=[]>
848  : MVE_comp<NoItinerary, iname, suffix, "", pattern> {
849
850  let Inst{28} = 0b1;
851  let Inst{25-24} = 0b11;
852  let Inst{23} = 0b0;
853  let Inst{21} = bit_21;
854  let Inst{20} = sz;
855  let Inst{11} = 0b1;
856  let Inst{8} = 0b1;
857  let Inst{6} = 0b1;
858  let Inst{4} = 0b1;
859
860  let Predicates = [HasMVEFloat];
861}
862
863def MVE_VMAXNMf32 : MVE_VMINMAXNM<"vmaxnm", "f32", 0b0, 0b0>;
864def MVE_VMAXNMf16 : MVE_VMINMAXNM<"vmaxnm", "f16", 0b1, 0b0>;
865
866let Predicates = [HasMVEFloat] in {
867  def : Pat<(v4f32 (fmaxnum (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
868            (v4f32 (MVE_VMAXNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
869  def : Pat<(v8f16 (fmaxnum (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
870            (v8f16 (MVE_VMAXNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
871}
872
873def MVE_VMINNMf32 : MVE_VMINMAXNM<"vminnm", "f32", 0b0, 0b1>;
874def MVE_VMINNMf16 : MVE_VMINMAXNM<"vminnm", "f16", 0b1, 0b1>;
875
876let Predicates = [HasMVEFloat] in {
877  def : Pat<(v4f32 (fminnum (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
878            (v4f32 (MVE_VMINNMf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
879  def : Pat<(v8f16 (fminnum (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
880            (v8f16 (MVE_VMINNMf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
881}
882
883
884class MVE_VMINMAX<string iname, string suffix, bit U, bits<2> size,
885              bit bit_4, list<dag> pattern=[]>
886  : MVE_comp<NoItinerary, iname, suffix, "", pattern> {
887
888  let Inst{28} = U;
889  let Inst{25-24} = 0b11;
890  let Inst{23} = 0b0;
891  let Inst{21-20} = size{1-0};
892  let Inst{11} = 0b0;
893  let Inst{8} = 0b0;
894  let Inst{6} = 0b1;
895  let Inst{4} = bit_4;
896}
897
898multiclass MVE_VMINMAX_all_sizes<string iname, bit bit_4> {
899  def s8  : MVE_VMINMAX<iname, "s8",  0b0, 0b00, bit_4>;
900  def s16 : MVE_VMINMAX<iname, "s16", 0b0, 0b01, bit_4>;
901  def s32 : MVE_VMINMAX<iname, "s32", 0b0, 0b10, bit_4>;
902  def u8  : MVE_VMINMAX<iname, "u8",  0b1, 0b00, bit_4>;
903  def u16 : MVE_VMINMAX<iname, "u16", 0b1, 0b01, bit_4>;
904  def u32 : MVE_VMINMAX<iname, "u32", 0b1, 0b10, bit_4>;
905}
906
907defm MVE_VMAX : MVE_VMINMAX_all_sizes<"vmax", 0b0>;
908defm MVE_VMIN : MVE_VMINMAX_all_sizes<"vmin", 0b1>;
909
910let Predicates = [HasMVEInt] in {
911  def : Pat<(v16i8 (smin (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
912            (v16i8 (MVE_VMINs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
913  def : Pat<(v8i16 (smin (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
914            (v8i16 (MVE_VMINs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
915  def : Pat<(v4i32 (smin (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
916            (v4i32 (MVE_VMINs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
917
918  def : Pat<(v16i8 (smax (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
919            (v16i8 (MVE_VMAXs8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
920  def : Pat<(v8i16 (smax (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
921            (v8i16 (MVE_VMAXs16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
922  def : Pat<(v4i32 (smax (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
923            (v4i32 (MVE_VMAXs32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
924
925  def : Pat<(v16i8 (umin (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
926            (v16i8 (MVE_VMINu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
927  def : Pat<(v8i16 (umin (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
928            (v8i16 (MVE_VMINu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
929  def : Pat<(v4i32 (umin (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
930            (v4i32 (MVE_VMINu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
931
932  def : Pat<(v16i8 (umax (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
933            (v16i8 (MVE_VMAXu8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
934  def : Pat<(v8i16 (umax (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
935            (v8i16 (MVE_VMAXu16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
936  def : Pat<(v4i32 (umax (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
937            (v4i32 (MVE_VMAXu32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
938}
939
940// end of mve_comp instructions
941
942// start of mve_bit instructions
943
944class MVE_bit_arith<dag oops, dag iops, string iname, string suffix,
945                    string ops, string cstr, list<dag> pattern=[]>
946  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred_r, cstr, pattern> {
947  bits<4> Qd;
948  bits<4> Qm;
949
950  let Inst{22} = Qd{3};
951  let Inst{15-13} = Qd{2-0};
952  let Inst{5} = Qm{3};
953  let Inst{3-1} = Qm{2-0};
954}
955
956def MVE_VBIC : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
957                             "vbic", "", "$Qd, $Qn, $Qm", ""> {
958  bits<4> Qn;
959
960  let Inst{28} = 0b0;
961  let Inst{25-23} = 0b110;
962  let Inst{21-20} = 0b01;
963  let Inst{19-17} = Qn{2-0};
964  let Inst{16} = 0b0;
965  let Inst{12-8} = 0b00001;
966  let Inst{7} = Qn{3};
967  let Inst{6} = 0b1;
968  let Inst{4} = 0b1;
969  let Inst{0} = 0b0;
970}
971
972class MVE_VREV<string iname, string suffix, bits<2> size, bits<2> bit_8_7>
973  : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm), iname,
974                  suffix, "$Qd, $Qm", ""> {
975
976  let Inst{28} = 0b1;
977  let Inst{25-23} = 0b111;
978  let Inst{21-20} = 0b11;
979  let Inst{19-18} = size;
980  let Inst{17-16} = 0b00;
981  let Inst{12-9} = 0b0000;
982  let Inst{8-7} = bit_8_7;
983  let Inst{6} = 0b1;
984  let Inst{4} = 0b0;
985  let Inst{0} = 0b0;
986}
987
988def MVE_VREV64_8  : MVE_VREV<"vrev64", "8", 0b00, 0b00>;
989def MVE_VREV64_16 : MVE_VREV<"vrev64", "16", 0b01, 0b00>;
990def MVE_VREV64_32 : MVE_VREV<"vrev64", "32", 0b10, 0b00>;
991
992def MVE_VREV32_8  : MVE_VREV<"vrev32", "8", 0b00, 0b01>;
993def MVE_VREV32_16 : MVE_VREV<"vrev32", "16", 0b01, 0b01>;
994
995def MVE_VREV16_8  : MVE_VREV<"vrev16", "8", 0b00, 0b10>;
996
997let Predicates = [HasMVEInt] in {
998  def : Pat<(v4i32 (ARMvrev64 (v4i32 MQPR:$src))),
999            (v4i32 (MVE_VREV64_32 (v4i32 MQPR:$src)))>;
1000  def : Pat<(v8i16 (ARMvrev64 (v8i16 MQPR:$src))),
1001            (v8i16 (MVE_VREV64_16 (v8i16 MQPR:$src)))>;
1002  def : Pat<(v16i8 (ARMvrev64 (v16i8 MQPR:$src))),
1003            (v16i8 (MVE_VREV64_8  (v16i8 MQPR:$src)))>;
1004
1005  def : Pat<(v8i16 (ARMvrev32 (v8i16 MQPR:$src))),
1006            (v8i16 (MVE_VREV32_16 (v8i16 MQPR:$src)))>;
1007  def : Pat<(v16i8 (ARMvrev32 (v16i8 MQPR:$src))),
1008            (v16i8 (MVE_VREV32_8  (v16i8 MQPR:$src)))>;
1009
1010  def : Pat<(v16i8 (ARMvrev16 (v16i8 MQPR:$src))),
1011            (v16i8 (MVE_VREV16_8  (v16i8 MQPR:$src)))>;
1012
1013  def : Pat<(v4f32 (ARMvrev64 (v4f32 MQPR:$src))),
1014            (v4f32 (MVE_VREV64_32 (v4f32 MQPR:$src)))>;
1015  def : Pat<(v8f16 (ARMvrev64 (v8f16 MQPR:$src))),
1016            (v8f16 (MVE_VREV64_16 (v8f16 MQPR:$src)))>;
1017  def : Pat<(v8f16 (ARMvrev32 (v8f16 MQPR:$src))),
1018            (v8f16 (MVE_VREV32_16 (v8f16 MQPR:$src)))>;
1019}
1020
1021def MVE_VMVN : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qm),
1022                             "vmvn", "", "$Qd, $Qm", ""> {
1023  let Inst{28} = 0b1;
1024  let Inst{25-23} = 0b111;
1025  let Inst{21-16} = 0b110000;
1026  let Inst{12-6} = 0b0010111;
1027  let Inst{4} = 0b0;
1028  let Inst{0} = 0b0;
1029}
1030
1031let Predicates = [HasMVEInt] in {
1032  def : Pat<(v16i8 (vnotq  (v16i8 MQPR:$val1))),
1033            (v16i8 (MVE_VMVN (v16i8 MQPR:$val1)))>;
1034  def : Pat<(v8i16 (vnotq  (v8i16 MQPR:$val1))),
1035            (v8i16 (MVE_VMVN (v8i16 MQPR:$val1)))>;
1036  def : Pat<(v4i32 (vnotq  (v4i32 MQPR:$val1))),
1037            (v4i32 (MVE_VMVN (v4i32 MQPR:$val1)))>;
1038  def : Pat<(v2i64 (vnotq  (v2i64 MQPR:$val1))),
1039            (v2i64 (MVE_VMVN (v2i64 MQPR:$val1)))>;
1040}
1041
1042class MVE_bit_ops<string iname, bits<2> bit_21_20, bit bit_28>
1043  : MVE_bit_arith<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
1044                  iname, "", "$Qd, $Qn, $Qm", ""> {
1045  bits<4> Qn;
1046
1047  let Inst{28} = bit_28;
1048  let Inst{25-23} = 0b110;
1049  let Inst{21-20} = bit_21_20;
1050  let Inst{19-17} = Qn{2-0};
1051  let Inst{16} = 0b0;
1052  let Inst{12-8} = 0b00001;
1053  let Inst{7} = Qn{3};
1054  let Inst{6} = 0b1;
1055  let Inst{4} = 0b1;
1056  let Inst{0} = 0b0;
1057}
1058
1059def MVE_VEOR : MVE_bit_ops<"veor", 0b00, 0b1>;
1060def MVE_VORN : MVE_bit_ops<"vorn", 0b11, 0b0>;
1061def MVE_VORR : MVE_bit_ops<"vorr", 0b10, 0b0>;
1062def MVE_VAND : MVE_bit_ops<"vand", 0b00, 0b0>;
1063
1064// add ignored suffixes as aliases
1065
1066foreach s=["s8", "s16", "s32", "u8", "u16", "u32", "i8", "i16", "i32", "f16", "f32"] in {
1067  def : MVEInstAlias<"vbic${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1068        (MVE_VBIC MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1069  def : MVEInstAlias<"veor${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1070        (MVE_VEOR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1071  def : MVEInstAlias<"vorn${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1072        (MVE_VORN MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1073  def : MVEInstAlias<"vorr${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1074        (MVE_VORR MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1075  def : MVEInstAlias<"vand${vp}." # s # "\t$QdSrc, $QnSrc, $QmSrc",
1076        (MVE_VAND MQPR:$QdSrc, MQPR:$QnSrc, MQPR:$QmSrc, vpred_r:$vp)>;
1077}
1078
1079let Predicates = [HasMVEInt] in {
1080  def : Pat<(v16i8 (and (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1081            (v16i8 (MVE_VAND (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1082  def : Pat<(v8i16 (and (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1083            (v8i16 (MVE_VAND (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1084  def : Pat<(v4i32 (and (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
1085            (v4i32 (MVE_VAND (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1086  def : Pat<(v2i64 (and (v2i64 MQPR:$val1), (v2i64 MQPR:$val2))),
1087            (v2i64 (MVE_VAND (v2i64 MQPR:$val1), (v2i64 MQPR:$val2)))>;
1088
1089  def : Pat<(v16i8 (or (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1090            (v16i8 (MVE_VORR (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1091  def : Pat<(v8i16 (or (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1092            (v8i16 (MVE_VORR (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1093  def : Pat<(v4i32 (or (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
1094            (v4i32 (MVE_VORR (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1095  def : Pat<(v2i64 (or (v2i64 MQPR:$val1), (v2i64 MQPR:$val2))),
1096            (v2i64 (MVE_VORR (v2i64 MQPR:$val1), (v2i64 MQPR:$val2)))>;
1097
1098  def : Pat<(v16i8 (xor (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1099            (v16i8 (MVE_VEOR (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1100  def : Pat<(v8i16 (xor (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1101            (v8i16 (MVE_VEOR (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1102  def : Pat<(v4i32 (xor (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
1103            (v4i32 (MVE_VEOR (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1104  def : Pat<(v2i64 (xor (v2i64 MQPR:$val1), (v2i64 MQPR:$val2))),
1105            (v2i64 (MVE_VEOR (v2i64 MQPR:$val1), (v2i64 MQPR:$val2)))>;
1106
1107  def : Pat<(v16i8 (and (v16i8 MQPR:$val1), (vnotq MQPR:$val2))),
1108            (v16i8 (MVE_VBIC (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1109  def : Pat<(v8i16 (and (v8i16 MQPR:$val1), (vnotq MQPR:$val2))),
1110            (v8i16 (MVE_VBIC (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1111  def : Pat<(v4i32 (and (v4i32 MQPR:$val1), (vnotq MQPR:$val2))),
1112            (v4i32 (MVE_VBIC (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1113  def : Pat<(v2i64 (and (v2i64 MQPR:$val1), (vnotq MQPR:$val2))),
1114            (v2i64 (MVE_VBIC (v2i64 MQPR:$val1), (v2i64 MQPR:$val2)))>;
1115
1116  def : Pat<(v16i8 (or (v16i8 MQPR:$val1), (vnotq MQPR:$val2))),
1117            (v16i8 (MVE_VORN (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1118  def : Pat<(v8i16 (or (v8i16 MQPR:$val1), (vnotq MQPR:$val2))),
1119            (v8i16 (MVE_VORN (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1120  def : Pat<(v4i32 (or (v4i32 MQPR:$val1), (vnotq MQPR:$val2))),
1121            (v4i32 (MVE_VORN (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1122  def : Pat<(v2i64 (or (v2i64 MQPR:$val1), (vnotq MQPR:$val2))),
1123            (v2i64 (MVE_VORN (v2i64 MQPR:$val1), (v2i64 MQPR:$val2)))>;
1124}
1125
1126class MVE_bit_cmode<string iname, string suffix, bits<4> cmode, dag inOps>
1127  : MVE_p<(outs MQPR:$Qd), inOps, NoItinerary,
1128          iname, suffix, "$Qd, $imm", vpred_n, "$Qd = $Qd_src"> {
1129  bits<8> imm;
1130  bits<4> Qd;
1131
1132  let Inst{28} = imm{7};
1133  let Inst{27-23} = 0b11111;
1134  let Inst{22} = Qd{3};
1135  let Inst{21-19} = 0b000;
1136  let Inst{18-16} = imm{6-4};
1137  let Inst{15-13} = Qd{2-0};
1138  let Inst{12} = 0b0;
1139  let Inst{11-8} = cmode;
1140  let Inst{7-6} = 0b01;
1141  let Inst{4} = 0b1;
1142  let Inst{3-0} = imm{3-0};
1143}
1144
1145class MVE_VORR<string suffix, bits<4> cmode, ExpandImm imm_type>
1146  : MVE_bit_cmode<"vorr", suffix, cmode, (ins MQPR:$Qd_src, imm_type:$imm)> {
1147  let Inst{5} = 0b0;
1148}
1149
1150def MVE_VORRIZ0v4i32  : MVE_VORR<"i32", 0b0001, expzero00>;
1151def MVE_VORRIZ0v8i16  : MVE_VORR<"i16", 0b1001, expzero00>;
1152def MVE_VORRIZ8v4i32  : MVE_VORR<"i32", 0b0011, expzero08>;
1153def MVE_VORRIZ8v8i16  : MVE_VORR<"i16", 0b1011, expzero08>;
1154def MVE_VORRIZ16v4i32 : MVE_VORR<"i32", 0b0101, expzero16>;
1155def MVE_VORRIZ24v4i32 : MVE_VORR<"i32", 0b0111, expzero24>;
1156
1157def MVE_VORNIZ0v4i32 : MVEAsmPseudo<"vorn${vp}.i32\t$Qd, $imm",
1158    (ins MQPR:$Qd_src, expzero00inv32:$imm, vpred_n:$vp), (outs MQPR:$Qd)>;
1159def MVE_VORNIZ0v8i16 : MVEAsmPseudo<"vorn${vp}.i16\t$Qd, $imm",
1160    (ins MQPR:$Qd_src, expzero00inv16:$imm, vpred_n:$vp), (outs MQPR:$Qd)>;
1161def MVE_VORNIZ8v4i32 : MVEAsmPseudo<"vorn${vp}.i32\t$Qd, $imm",
1162    (ins MQPR:$Qd_src, expzero08inv32:$imm, vpred_n:$vp), (outs MQPR:$Qd)>;
1163def MVE_VORNIZ8v8i16 : MVEAsmPseudo<"vorn${vp}.i16\t$Qd, $imm",
1164    (ins MQPR:$Qd_src, expzero08inv16:$imm, vpred_n:$vp), (outs MQPR:$Qd)>;
1165def MVE_VORNIZ16v4i32 : MVEAsmPseudo<"vorn${vp}.i32\t$Qd, $imm",
1166    (ins MQPR:$Qd_src, expzero16inv32:$imm, vpred_n:$vp), (outs MQPR:$Qd)>;
1167def MVE_VORNIZ24v4i32 : MVEAsmPseudo<"vorn${vp}.i32\t$Qd, $imm",
1168    (ins MQPR:$Qd_src, expzero24inv32:$imm, vpred_n:$vp), (outs MQPR:$Qd)>;
1169
1170def MVE_VMOV : MVEInstAlias<"vmov${vp}\t$Qd, $Qm",
1171    (MVE_VORR MQPR:$Qd, MQPR:$Qm, MQPR:$Qm, vpred_r:$vp)>;
1172
1173class MVE_VBIC<string suffix, bits<4> cmode, ExpandImm imm_type>
1174  : MVE_bit_cmode<"vbic", suffix, cmode, (ins MQPR:$Qd_src, imm_type:$imm)> {
1175  let Inst{5} = 0b1;
1176}
1177
1178def MVE_VBICIZ0v4i32  : MVE_VBIC<"i32", 0b0001, expzero00>;
1179def MVE_VBICIZ0v8i16  : MVE_VBIC<"i16", 0b1001, expzero00>;
1180def MVE_VBICIZ8v4i32  : MVE_VBIC<"i32", 0b0011, expzero08>;
1181def MVE_VBICIZ8v8i16  : MVE_VBIC<"i16", 0b1011, expzero08>;
1182def MVE_VBICIZ16v4i32 : MVE_VBIC<"i32", 0b0101, expzero16>;
1183def MVE_VBICIZ24v4i32 : MVE_VBIC<"i32", 0b0111, expzero24>;
1184
1185def MVE_VANDIZ0v4i32 : MVEAsmPseudo<"vand${vp}.i32\t$Qda, $imm",
1186    (ins MQPR:$Qda_src, expzero00inv32:$imm, vpred_n:$vp), (outs MQPR:$Qda)>;
1187def MVE_VANDIZ0v8i16 : MVEAsmPseudo<"vand${vp}.i16\t$Qda, $imm",
1188    (ins MQPR:$Qda_src, expzero00inv16:$imm, vpred_n:$vp), (outs MQPR:$Qda)>;
1189def MVE_VANDIZ8v4i32 : MVEAsmPseudo<"vand${vp}.i32\t$Qda, $imm",
1190    (ins MQPR:$Qda_src, expzero08inv32:$imm, vpred_n:$vp), (outs MQPR:$Qda)>;
1191def MVE_VANDIZ8v8i16 : MVEAsmPseudo<"vand${vp}.i16\t$Qda, $imm",
1192    (ins MQPR:$Qda_src, expzero08inv16:$imm, vpred_n:$vp), (outs MQPR:$Qda)>;
1193def MVE_VANDIZ16v4i32 : MVEAsmPseudo<"vand${vp}.i32\t$Qda, $imm",
1194    (ins MQPR:$Qda_src, expzero16inv32:$imm, vpred_n:$vp), (outs MQPR:$Qda)>;
1195def MVE_VANDIZ24v4i32 : MVEAsmPseudo<"vand${vp}.i32\t$Qda, $imm",
1196    (ins MQPR:$Qda_src, expzero24inv32:$imm, vpred_n:$vp), (outs MQPR:$Qda)>;
1197
1198class MVE_VMOV_lane_direction {
1199  bit bit_20;
1200  dag oops;
1201  dag iops;
1202  string ops;
1203  string cstr;
1204}
1205def MVE_VMOV_from_lane : MVE_VMOV_lane_direction {
1206  let bit_20 = 0b1;
1207  let oops = (outs rGPR:$Rt);
1208  let iops = (ins MQPR:$Qd);
1209  let ops = "$Rt, $Qd$Idx";
1210  let cstr = "";
1211}
1212def MVE_VMOV_to_lane : MVE_VMOV_lane_direction {
1213  let bit_20 = 0b0;
1214  let oops = (outs MQPR:$Qd);
1215  let iops = (ins MQPR:$Qd_src, rGPR:$Rt);
1216  let ops = "$Qd$Idx, $Rt";
1217  let cstr = "$Qd = $Qd_src";
1218}
1219
1220class MVE_VMOV_lane<string suffix, bit U, dag indexop,
1221                    MVE_VMOV_lane_direction dir>
1222  : MVE_VMOV_lane_base<dir.oops, !con(dir.iops, indexop), NoItinerary,
1223                       "vmov", suffix, dir.ops, dir.cstr, []> {
1224  bits<4> Qd;
1225  bits<4> Rt;
1226
1227  let Inst{31-24} = 0b11101110;
1228  let Inst{23} = U;
1229  let Inst{20} = dir.bit_20;
1230  let Inst{19-17} = Qd{2-0};
1231  let Inst{15-12} = Rt{3-0};
1232  let Inst{11-8} = 0b1011;
1233  let Inst{7} = Qd{3};
1234  let Inst{4-0} = 0b10000;
1235}
1236
1237class MVE_VMOV_lane_32<MVE_VMOV_lane_direction dir>
1238    : MVE_VMOV_lane<"32", 0b0, (ins MVEVectorIndex<4>:$Idx), dir> {
1239  bits<2> Idx;
1240  let Inst{22} = 0b0;
1241  let Inst{6-5} = 0b00;
1242  let Inst{16} = Idx{1};
1243  let Inst{21} = Idx{0};
1244
1245  let Predicates = [HasFPRegsV8_1M];
1246}
1247
1248class MVE_VMOV_lane_16<string suffix, bit U, MVE_VMOV_lane_direction dir>
1249  : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<8>:$Idx), dir> {
1250  bits<3> Idx;
1251  let Inst{22} = 0b0;
1252  let Inst{5} = 0b1;
1253  let Inst{16} = Idx{2};
1254  let Inst{21} = Idx{1};
1255  let Inst{6} = Idx{0};
1256}
1257
1258class MVE_VMOV_lane_8<string suffix, bit U, MVE_VMOV_lane_direction dir>
1259  : MVE_VMOV_lane<suffix, U, (ins MVEVectorIndex<16>:$Idx), dir> {
1260  bits<4> Idx;
1261  let Inst{22} = 0b1;
1262  let Inst{16} = Idx{3};
1263  let Inst{21} = Idx{2};
1264  let Inst{6} = Idx{1};
1265  let Inst{5} = Idx{0};
1266}
1267
1268def MVE_VMOV_from_lane_32  : MVE_VMOV_lane_32<            MVE_VMOV_from_lane>;
1269def MVE_VMOV_to_lane_32    : MVE_VMOV_lane_32<            MVE_VMOV_to_lane>;
1270def MVE_VMOV_from_lane_s16 : MVE_VMOV_lane_16<"s16", 0b0, MVE_VMOV_from_lane>;
1271def MVE_VMOV_from_lane_u16 : MVE_VMOV_lane_16<"u16", 0b1, MVE_VMOV_from_lane>;
1272def MVE_VMOV_to_lane_16    : MVE_VMOV_lane_16< "16", 0b0, MVE_VMOV_to_lane>;
1273def MVE_VMOV_from_lane_s8  : MVE_VMOV_lane_8 < "s8", 0b0, MVE_VMOV_from_lane>;
1274def MVE_VMOV_from_lane_u8  : MVE_VMOV_lane_8 < "u8", 0b1, MVE_VMOV_from_lane>;
1275def MVE_VMOV_to_lane_8     : MVE_VMOV_lane_8 <  "8", 0b0, MVE_VMOV_to_lane>;
1276
1277let Predicates = [HasMVEInt] in {
1278  def : Pat<(extractelt (v2f64 MQPR:$src), imm:$lane),
1279            (f64 (EXTRACT_SUBREG MQPR:$src, (DSubReg_f64_reg imm:$lane)))>;
1280  def : Pat<(insertelt (v2f64 MQPR:$src1), DPR:$src2, imm:$lane),
1281            (INSERT_SUBREG (v2f64 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), DPR:$src2, (DSubReg_f64_reg imm:$lane))>;
1282
1283  def : Pat<(extractelt (v4i32 MQPR:$src), imm:$lane),
1284            (COPY_TO_REGCLASS
1285              (i32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), rGPR)>;
1286  def : Pat<(insertelt (v4i32 MQPR:$src1), rGPR:$src2, imm:$lane),
1287            (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$src2, imm:$lane)>;
1288
1289  def : Pat<(vector_insert (v16i8 MQPR:$src1), rGPR:$src2, imm:$lane),
1290            (MVE_VMOV_to_lane_8  MQPR:$src1, rGPR:$src2, imm:$lane)>;
1291  def : Pat<(vector_insert (v8i16 MQPR:$src1), rGPR:$src2, imm:$lane),
1292            (MVE_VMOV_to_lane_16 MQPR:$src1, rGPR:$src2, imm:$lane)>;
1293
1294  def : Pat<(ARMvgetlanes (v16i8 MQPR:$src), imm:$lane),
1295            (MVE_VMOV_from_lane_s8 MQPR:$src, imm:$lane)>;
1296  def : Pat<(ARMvgetlanes (v8i16 MQPR:$src), imm:$lane),
1297            (MVE_VMOV_from_lane_s16 MQPR:$src, imm:$lane)>;
1298  def : Pat<(ARMvgetlaneu (v16i8 MQPR:$src), imm:$lane),
1299            (MVE_VMOV_from_lane_u8 MQPR:$src, imm:$lane)>;
1300  def : Pat<(ARMvgetlaneu (v8i16 MQPR:$src), imm:$lane),
1301            (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane)>;
1302
1303  def : Pat<(v16i8 (scalar_to_vector GPR:$src)),
1304            (MVE_VMOV_to_lane_8  (v16i8 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1305  def : Pat<(v8i16 (scalar_to_vector GPR:$src)),
1306            (MVE_VMOV_to_lane_16 (v8i16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1307  def : Pat<(v4i32 (scalar_to_vector GPR:$src)),
1308            (MVE_VMOV_to_lane_32 (v4i32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1309
1310  // Floating point patterns, still enabled under HasMVEInt
1311  def : Pat<(extractelt (v4f32 MQPR:$src), imm:$lane),
1312            (COPY_TO_REGCLASS (f32 (EXTRACT_SUBREG MQPR:$src, (SSubReg_f32_reg imm:$lane))), SPR)>;
1313  def : Pat<(insertelt (v4f32 MQPR:$src1), (f32 SPR:$src2), imm:$lane),
1314            (INSERT_SUBREG (v4f32 (COPY_TO_REGCLASS MQPR:$src1, MQPR)), SPR:$src2, (SSubReg_f32_reg imm:$lane))>;
1315
1316  def : Pat<(insertelt (v8f16 MQPR:$src1), HPR:$src2, imm:$lane),
1317            (MVE_VMOV_to_lane_16 MQPR:$src1, (COPY_TO_REGCLASS HPR:$src2, rGPR), imm:$lane)>;
1318  def : Pat<(extractelt (v8f16 MQPR:$src), imm:$lane),
1319            (COPY_TO_REGCLASS (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane), HPR)>;
1320
1321  def : Pat<(v4f32 (scalar_to_vector SPR:$src)),
1322            (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), SPR:$src, ssub_0)>;
1323  def : Pat<(v4f32 (scalar_to_vector GPR:$src)),
1324            (MVE_VMOV_to_lane_32 (v4f32 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1325  def : Pat<(v8f16 (scalar_to_vector HPR:$src)),
1326            (INSERT_SUBREG (v8f16 (IMPLICIT_DEF)), HPR:$src, ssub_0)>;
1327  def : Pat<(v8f16 (scalar_to_vector GPR:$src)),
1328            (MVE_VMOV_to_lane_16 (v8f16 (IMPLICIT_DEF)), rGPR:$src, (i32 0))>;
1329}
1330
1331// end of mve_bit instructions
1332
1333// start of MVE Integer instructions
1334
1335class MVE_int<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
1336  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
1337          iname, suffix, "$Qd, $Qn, $Qm", vpred_r, "", pattern> {
1338  bits<4> Qd;
1339  bits<4> Qn;
1340  bits<4> Qm;
1341
1342  let Inst{22} = Qd{3};
1343  let Inst{21-20} = size;
1344  let Inst{19-17} = Qn{2-0};
1345  let Inst{15-13} = Qd{2-0};
1346  let Inst{7} = Qn{3};
1347  let Inst{6} = 0b1;
1348  let Inst{5} = Qm{3};
1349  let Inst{3-1} = Qm{2-0};
1350}
1351
1352class MVE_VMULt1<string suffix, bits<2> size, list<dag> pattern=[]>
1353  : MVE_int<"vmul", suffix, size, pattern> {
1354
1355  let Inst{28} = 0b0;
1356  let Inst{25-23} = 0b110;
1357  let Inst{16} = 0b0;
1358  let Inst{12-8} = 0b01001;
1359  let Inst{4} = 0b1;
1360  let Inst{0} = 0b0;
1361}
1362
1363def MVE_VMULt1i8  : MVE_VMULt1<"i8", 0b00>;
1364def MVE_VMULt1i16 : MVE_VMULt1<"i16", 0b01>;
1365def MVE_VMULt1i32 : MVE_VMULt1<"i32", 0b10>;
1366
1367let Predicates = [HasMVEInt] in {
1368  def : Pat<(v16i8 (mul (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1369            (v16i8 (MVE_VMULt1i8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1370  def : Pat<(v8i16 (mul (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1371            (v8i16 (MVE_VMULt1i16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1372  def : Pat<(v4i32 (mul (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
1373            (v4i32 (MVE_VMULt1i32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1374}
1375
1376class MVE_VQxDMULH<string iname, string suffix, bits<2> size, bit rounding,
1377                  list<dag> pattern=[]>
1378  : MVE_int<iname, suffix, size, pattern> {
1379
1380  let Inst{28} = rounding;
1381  let Inst{25-23} = 0b110;
1382  let Inst{16} = 0b0;
1383  let Inst{12-8} = 0b01011;
1384  let Inst{4} = 0b0;
1385  let Inst{0} = 0b0;
1386}
1387
1388class MVE_VQDMULH<string suffix, bits<2> size, list<dag> pattern=[]>
1389  : MVE_VQxDMULH<"vqdmulh", suffix, size, 0b0, pattern>;
1390class MVE_VQRDMULH<string suffix, bits<2> size, list<dag> pattern=[]>
1391  : MVE_VQxDMULH<"vqrdmulh", suffix, size, 0b1, pattern>;
1392
1393def MVE_VQDMULHi8   : MVE_VQDMULH<"s8",  0b00>;
1394def MVE_VQDMULHi16  : MVE_VQDMULH<"s16", 0b01>;
1395def MVE_VQDMULHi32  : MVE_VQDMULH<"s32", 0b10>;
1396
1397def MVE_VQRDMULHi8  : MVE_VQRDMULH<"s8",  0b00>;
1398def MVE_VQRDMULHi16 : MVE_VQRDMULH<"s16", 0b01>;
1399def MVE_VQRDMULHi32 : MVE_VQRDMULH<"s32", 0b10>;
1400
1401class MVE_VADDSUB<string iname, string suffix, bits<2> size, bit subtract,
1402                    list<dag> pattern=[]>
1403  : MVE_int<iname, suffix, size, pattern> {
1404
1405  let Inst{28} = subtract;
1406  let Inst{25-23} = 0b110;
1407  let Inst{16} = 0b0;
1408  let Inst{12-8} = 0b01000;
1409  let Inst{4} = 0b0;
1410  let Inst{0} = 0b0;
1411}
1412
1413class MVE_VADD<string suffix, bits<2> size, list<dag> pattern=[]>
1414  : MVE_VADDSUB<"vadd", suffix, size, 0b0, pattern>;
1415class MVE_VSUB<string suffix, bits<2> size, list<dag> pattern=[]>
1416  : MVE_VADDSUB<"vsub", suffix, size, 0b1, pattern>;
1417
1418def MVE_VADDi8  : MVE_VADD<"i8",  0b00>;
1419def MVE_VADDi16 : MVE_VADD<"i16", 0b01>;
1420def MVE_VADDi32 : MVE_VADD<"i32", 0b10>;
1421
1422let Predicates = [HasMVEInt] in {
1423  def : Pat<(v16i8 (add (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1424            (v16i8 (MVE_VADDi8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1425  def : Pat<(v8i16 (add (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1426            (v8i16 (MVE_VADDi16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1427  def : Pat<(v4i32 (add (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
1428            (v4i32 (MVE_VADDi32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1429}
1430
1431def MVE_VSUBi8  : MVE_VSUB<"i8",  0b00>;
1432def MVE_VSUBi16 : MVE_VSUB<"i16", 0b01>;
1433def MVE_VSUBi32 : MVE_VSUB<"i32", 0b10>;
1434
1435let Predicates = [HasMVEInt] in {
1436  def : Pat<(v16i8 (sub (v16i8 MQPR:$val1), (v16i8 MQPR:$val2))),
1437            (v16i8 (MVE_VSUBi8 (v16i8 MQPR:$val1), (v16i8 MQPR:$val2)))>;
1438  def : Pat<(v8i16 (sub (v8i16 MQPR:$val1), (v8i16 MQPR:$val2))),
1439            (v8i16 (MVE_VSUBi16 (v8i16 MQPR:$val1), (v8i16 MQPR:$val2)))>;
1440  def : Pat<(v4i32 (sub (v4i32 MQPR:$val1), (v4i32 MQPR:$val2))),
1441            (v4i32 (MVE_VSUBi32 (v4i32 MQPR:$val1), (v4i32 MQPR:$val2)))>;
1442}
1443
1444class MVE_VQADDSUB<string iname, string suffix, bit U, bit subtract,
1445                   bits<2> size, list<dag> pattern=[]>
1446  : MVE_int<iname, suffix, size, pattern> {
1447
1448  let Inst{28} = U;
1449  let Inst{25-23} = 0b110;
1450  let Inst{16} = 0b0;
1451  let Inst{12-10} = 0b000;
1452  let Inst{9} = subtract;
1453  let Inst{8} = 0b0;
1454  let Inst{4} = 0b1;
1455  let Inst{0} = 0b0;
1456}
1457
1458class MVE_VQADD<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
1459  : MVE_VQADDSUB<"vqadd", suffix, U, 0b0, size, pattern>;
1460class MVE_VQSUB<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
1461  : MVE_VQADDSUB<"vqsub", suffix, U, 0b1, size, pattern>;
1462
1463def MVE_VQADDs8  : MVE_VQADD<"s8",  0b0, 0b00>;
1464def MVE_VQADDs16 : MVE_VQADD<"s16", 0b0, 0b01>;
1465def MVE_VQADDs32 : MVE_VQADD<"s32", 0b0, 0b10>;
1466def MVE_VQADDu8  : MVE_VQADD<"u8",  0b1, 0b00>;
1467def MVE_VQADDu16 : MVE_VQADD<"u16", 0b1, 0b01>;
1468def MVE_VQADDu32 : MVE_VQADD<"u32", 0b1, 0b10>;
1469
1470def MVE_VQSUBs8  : MVE_VQSUB<"s8",  0b0, 0b00>;
1471def MVE_VQSUBs16 : MVE_VQSUB<"s16", 0b0, 0b01>;
1472def MVE_VQSUBs32 : MVE_VQSUB<"s32", 0b0, 0b10>;
1473def MVE_VQSUBu8  : MVE_VQSUB<"u8",  0b1, 0b00>;
1474def MVE_VQSUBu16 : MVE_VQSUB<"u16", 0b1, 0b01>;
1475def MVE_VQSUBu32 : MVE_VQSUB<"u32", 0b1, 0b10>;
1476
1477class MVE_VABD_int<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
1478  : MVE_int<"vabd", suffix, size, pattern> {
1479
1480  let Inst{28} = U;
1481  let Inst{25-23} = 0b110;
1482  let Inst{16} = 0b0;
1483  let Inst{12-8} = 0b00111;
1484  let Inst{4} = 0b0;
1485  let Inst{0} = 0b0;
1486}
1487
1488def MVE_VABDs8  : MVE_VABD_int<"s8", 0b0, 0b00>;
1489def MVE_VABDs16 : MVE_VABD_int<"s16", 0b0, 0b01>;
1490def MVE_VABDs32 : MVE_VABD_int<"s32", 0b0, 0b10>;
1491def MVE_VABDu8  : MVE_VABD_int<"u8", 0b1, 0b00>;
1492def MVE_VABDu16 : MVE_VABD_int<"u16", 0b1, 0b01>;
1493def MVE_VABDu32 : MVE_VABD_int<"u32", 0b1, 0b10>;
1494
1495class MVE_VRHADD<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
1496  : MVE_int<"vrhadd", suffix, size, pattern> {
1497
1498  let Inst{28} = U;
1499  let Inst{25-23} = 0b110;
1500  let Inst{16} = 0b0;
1501  let Inst{12-8} = 0b00001;
1502  let Inst{4} = 0b0;
1503  let Inst{0} = 0b0;
1504}
1505
1506def MVE_VRHADDs8  : MVE_VRHADD<"s8", 0b0, 0b00>;
1507def MVE_VRHADDs16 : MVE_VRHADD<"s16", 0b0, 0b01>;
1508def MVE_VRHADDs32 : MVE_VRHADD<"s32", 0b0, 0b10>;
1509def MVE_VRHADDu8  : MVE_VRHADD<"u8", 0b1, 0b00>;
1510def MVE_VRHADDu16 : MVE_VRHADD<"u16", 0b1, 0b01>;
1511def MVE_VRHADDu32 : MVE_VRHADD<"u32", 0b1, 0b10>;
1512
1513class MVE_VHADDSUB<string iname, string suffix, bit U, bit subtract,
1514                   bits<2> size, list<dag> pattern=[]>
1515  : MVE_int<iname, suffix, size, pattern> {
1516
1517  let Inst{28} = U;
1518  let Inst{25-23} = 0b110;
1519  let Inst{16} = 0b0;
1520  let Inst{12-10} = 0b000;
1521  let Inst{9} = subtract;
1522  let Inst{8} = 0b0;
1523  let Inst{4} = 0b0;
1524  let Inst{0} = 0b0;
1525}
1526
1527class MVE_VHADD<string suffix, bit U, bits<2> size,
1528              list<dag> pattern=[]>
1529  : MVE_VHADDSUB<"vhadd", suffix, U, 0b0, size, pattern>;
1530class MVE_VHSUB<string suffix, bit U, bits<2> size,
1531              list<dag> pattern=[]>
1532  : MVE_VHADDSUB<"vhsub", suffix, U, 0b1, size, pattern>;
1533
1534def MVE_VHADDs8  : MVE_VHADD<"s8",  0b0, 0b00>;
1535def MVE_VHADDs16 : MVE_VHADD<"s16", 0b0, 0b01>;
1536def MVE_VHADDs32 : MVE_VHADD<"s32", 0b0, 0b10>;
1537def MVE_VHADDu8  : MVE_VHADD<"u8",  0b1, 0b00>;
1538def MVE_VHADDu16 : MVE_VHADD<"u16", 0b1, 0b01>;
1539def MVE_VHADDu32 : MVE_VHADD<"u32", 0b1, 0b10>;
1540
1541def MVE_VHSUBs8  : MVE_VHSUB<"s8",  0b0, 0b00>;
1542def MVE_VHSUBs16 : MVE_VHSUB<"s16", 0b0, 0b01>;
1543def MVE_VHSUBs32 : MVE_VHSUB<"s32", 0b0, 0b10>;
1544def MVE_VHSUBu8  : MVE_VHSUB<"u8",  0b1, 0b00>;
1545def MVE_VHSUBu16 : MVE_VHSUB<"u16", 0b1, 0b01>;
1546def MVE_VHSUBu32 : MVE_VHSUB<"u32", 0b1, 0b10>;
1547
1548class MVE_VDUP<string suffix, bit B, bit E, list<dag> pattern=[]>
1549  : MVE_p<(outs MQPR:$Qd), (ins rGPR:$Rt), NoItinerary,
1550          "vdup", suffix, "$Qd, $Rt", vpred_r, "", pattern> {
1551  bits<4> Qd;
1552  bits<4> Rt;
1553
1554  let Inst{28} = 0b0;
1555  let Inst{25-23} = 0b101;
1556  let Inst{22} = B;
1557  let Inst{21-20} = 0b10;
1558  let Inst{19-17} = Qd{2-0};
1559  let Inst{16} = 0b0;
1560  let Inst{15-12} = Rt;
1561  let Inst{11-8} = 0b1011;
1562  let Inst{7} = Qd{3};
1563  let Inst{6} = 0b0;
1564  let Inst{5} = E;
1565  let Inst{4-0} = 0b10000;
1566}
1567
1568def MVE_VDUP32 : MVE_VDUP<"32", 0b0, 0b0>;
1569def MVE_VDUP16 : MVE_VDUP<"16", 0b0, 0b1>;
1570def MVE_VDUP8  : MVE_VDUP<"8",  0b1, 0b0>;
1571
1572let Predicates = [HasMVEInt] in {
1573  def : Pat<(v16i8 (ARMvdup (i32 rGPR:$elem))),
1574            (MVE_VDUP8  rGPR:$elem)>;
1575  def : Pat<(v8i16 (ARMvdup (i32 rGPR:$elem))),
1576            (MVE_VDUP16 rGPR:$elem)>;
1577  def : Pat<(v4i32 (ARMvdup (i32 rGPR:$elem))),
1578            (MVE_VDUP32 rGPR:$elem)>;
1579
1580  def : Pat<(v4i32 (ARMvduplane (v4i32 MQPR:$src), imm:$lane)),
1581            (MVE_VDUP32 (MVE_VMOV_from_lane_32 MQPR:$src, imm:$lane))>;
1582  // For the 16-bit and 8-bit vduplanes we don't care about the signedness
1583  // of the lane move operation as we only want the lowest 8/16 bits anyway.
1584  def : Pat<(v8i16 (ARMvduplane (v8i16 MQPR:$src), imm:$lane)),
1585            (MVE_VDUP16 (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane))>;
1586  def : Pat<(v16i8 (ARMvduplane (v16i8 MQPR:$src), imm:$lane)),
1587            (MVE_VDUP8  (MVE_VMOV_from_lane_u8 MQPR:$src, imm:$lane))>;
1588
1589  def : Pat<(v4f32 (ARMvdup (f32 SPR:$elem))),
1590            (v4f32 (MVE_VDUP32 (i32 (COPY_TO_REGCLASS (f32 SPR:$elem), rGPR))))>;
1591  def : Pat<(v8f16 (ARMvdup (f16 HPR:$elem))),
1592            (v8f16 (MVE_VDUP16 (i32 (COPY_TO_REGCLASS (f16 HPR:$elem), rGPR))))>;
1593
1594  def : Pat<(v4f32 (ARMvduplane (v4f32 MQPR:$src), imm:$lane)),
1595            (MVE_VDUP32 (MVE_VMOV_from_lane_32 MQPR:$src, imm:$lane))>;
1596  def : Pat<(v8f16 (ARMvduplane (v8f16 MQPR:$src), imm:$lane)),
1597            (MVE_VDUP16 (MVE_VMOV_from_lane_u16 MQPR:$src, imm:$lane))>;
1598}
1599
1600
1601class MVEIntSingleSrc<string iname, string suffix, bits<2> size,
1602                         list<dag> pattern=[]>
1603  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm), NoItinerary,
1604          iname, suffix, "$Qd, $Qm", vpred_r, "", pattern> {
1605  bits<4> Qd;
1606  bits<4> Qm;
1607
1608  let Inst{22} = Qd{3};
1609  let Inst{19-18} = size{1-0};
1610  let Inst{15-13} = Qd{2-0};
1611  let Inst{5} = Qm{3};
1612  let Inst{3-1} = Qm{2-0};
1613}
1614
1615class MVE_VCLSCLZ<string iname, string suffix, bits<2> size,
1616                   bit count_zeroes, list<dag> pattern=[]>
1617  : MVEIntSingleSrc<iname, suffix, size, pattern> {
1618
1619  let Inst{28} = 0b1;
1620  let Inst{25-23} = 0b111;
1621  let Inst{21-20} = 0b11;
1622  let Inst{17-16} = 0b00;
1623  let Inst{12-8} = 0b00100;
1624  let Inst{7} = count_zeroes;
1625  let Inst{6} = 0b1;
1626  let Inst{4} = 0b0;
1627  let Inst{0} = 0b0;
1628}
1629
1630def MVE_VCLSs8  : MVE_VCLSCLZ<"vcls", "s8",  0b00, 0b0>;
1631def MVE_VCLSs16 : MVE_VCLSCLZ<"vcls", "s16", 0b01, 0b0>;
1632def MVE_VCLSs32 : MVE_VCLSCLZ<"vcls", "s32", 0b10, 0b0>;
1633
1634def MVE_VCLZs8  : MVE_VCLSCLZ<"vclz", "i8",  0b00, 0b1>;
1635def MVE_VCLZs16 : MVE_VCLSCLZ<"vclz", "i16", 0b01, 0b1>;
1636def MVE_VCLZs32 : MVE_VCLSCLZ<"vclz", "i32", 0b10, 0b1>;
1637
1638class MVE_VABSNEG_int<string iname, string suffix, bits<2> size, bit negate,
1639                      list<dag> pattern=[]>
1640  : MVEIntSingleSrc<iname, suffix, size, pattern> {
1641
1642  let Inst{28} = 0b1;
1643  let Inst{25-23} = 0b111;
1644  let Inst{21-20} = 0b11;
1645  let Inst{17-16} = 0b01;
1646  let Inst{12-8} = 0b00011;
1647  let Inst{7} = negate;
1648  let Inst{6} = 0b1;
1649  let Inst{4} = 0b0;
1650  let Inst{0} = 0b0;
1651}
1652
1653def MVE_VABSs8  : MVE_VABSNEG_int<"vabs", "s8",  0b00, 0b0>;
1654def MVE_VABSs16 : MVE_VABSNEG_int<"vabs", "s16", 0b01, 0b0>;
1655def MVE_VABSs32 : MVE_VABSNEG_int<"vabs", "s32", 0b10, 0b0>;
1656
1657let Predicates = [HasMVEInt] in {
1658  def : Pat<(v16i8 (abs (v16i8 MQPR:$v))),
1659            (v16i8 (MVE_VABSs8 $v))>;
1660  def : Pat<(v8i16 (abs (v8i16 MQPR:$v))),
1661            (v8i16 (MVE_VABSs16 $v))>;
1662  def : Pat<(v4i32 (abs (v4i32 MQPR:$v))),
1663            (v4i32 (MVE_VABSs32 $v))>;
1664}
1665
1666def MVE_VNEGs8  : MVE_VABSNEG_int<"vneg", "s8",  0b00, 0b1>;
1667def MVE_VNEGs16 : MVE_VABSNEG_int<"vneg", "s16", 0b01, 0b1>;
1668def MVE_VNEGs32 : MVE_VABSNEG_int<"vneg", "s32", 0b10, 0b1>;
1669
1670let Predicates = [HasMVEInt] in {
1671  def : Pat<(v16i8 (vnegq (v16i8 MQPR:$v))),
1672            (v16i8 (MVE_VNEGs8 $v))>;
1673  def : Pat<(v8i16 (vnegq (v8i16 MQPR:$v))),
1674            (v8i16 (MVE_VNEGs16 $v))>;
1675  def : Pat<(v4i32 (vnegq (v4i32 MQPR:$v))),
1676            (v4i32 (MVE_VNEGs32 $v))>;
1677}
1678
1679class MVE_VQABSNEG<string iname, string suffix, bits<2> size,
1680                   bit negate, list<dag> pattern=[]>
1681  : MVEIntSingleSrc<iname, suffix, size, pattern> {
1682
1683  let Inst{28} = 0b1;
1684  let Inst{25-23} = 0b111;
1685  let Inst{21-20} = 0b11;
1686  let Inst{17-16} = 0b00;
1687  let Inst{12-8} = 0b00111;
1688  let Inst{7} = negate;
1689  let Inst{6} = 0b1;
1690  let Inst{4} = 0b0;
1691  let Inst{0} = 0b0;
1692}
1693
1694def MVE_VQABSs8  : MVE_VQABSNEG<"vqabs", "s8",  0b00, 0b0>;
1695def MVE_VQABSs16 : MVE_VQABSNEG<"vqabs", "s16", 0b01, 0b0>;
1696def MVE_VQABSs32 : MVE_VQABSNEG<"vqabs", "s32", 0b10, 0b0>;
1697
1698def MVE_VQNEGs8  : MVE_VQABSNEG<"vqneg", "s8",  0b00, 0b1>;
1699def MVE_VQNEGs16 : MVE_VQABSNEG<"vqneg", "s16", 0b01, 0b1>;
1700def MVE_VQNEGs32 : MVE_VQABSNEG<"vqneg", "s32", 0b10, 0b1>;
1701
1702class MVE_mod_imm<string iname, string suffix, bits<4> cmode, bit op,
1703                  dag iops, list<dag> pattern=[]>
1704  : MVE_p<(outs MQPR:$Qd), iops, NoItinerary, iname, suffix, "$Qd, $imm",
1705          vpred_r, "", pattern> {
1706  bits<13> imm;
1707  bits<4> Qd;
1708
1709  let Inst{28} = imm{7};
1710  let Inst{25-23} = 0b111;
1711  let Inst{22} = Qd{3};
1712  let Inst{21-19} = 0b000;
1713  let Inst{18-16} = imm{6-4};
1714  let Inst{15-13} = Qd{2-0};
1715  let Inst{12} = 0b0;
1716  let Inst{11-8} = cmode{3-0};
1717  let Inst{7-6} = 0b01;
1718  let Inst{5} = op;
1719  let Inst{4} = 0b1;
1720  let Inst{3-0} = imm{3-0};
1721
1722  let DecoderMethod = "DecodeMVEModImmInstruction";
1723}
1724
1725let isReMaterializable = 1 in {
1726let isAsCheapAsAMove = 1 in {
1727def MVE_VMOVimmi8  : MVE_mod_imm<"vmov", "i8",  {1,1,1,0}, 0b0, (ins nImmSplatI8:$imm)>;
1728def MVE_VMOVimmi16 : MVE_mod_imm<"vmov", "i16", {1,0,?,0}, 0b0, (ins nImmSplatI16:$imm)> {
1729  let Inst{9} = imm{9};
1730}
1731def MVE_VMOVimmi32 : MVE_mod_imm<"vmov", "i32", {?,?,?,?}, 0b0, (ins nImmVMOVI32:$imm)> {
1732  let Inst{11-8} = imm{11-8};
1733}
1734def MVE_VMOVimmi64 : MVE_mod_imm<"vmov", "i64", {1,1,1,0}, 0b1, (ins nImmSplatI64:$imm)>;
1735def MVE_VMOVimmf32 : MVE_mod_imm<"vmov", "f32", {1,1,1,1}, 0b0, (ins nImmVMOVF32:$imm)>;
1736} // let isAsCheapAsAMove = 1
1737
1738def MVE_VMVNimmi16 : MVE_mod_imm<"vmvn", "i16", {1,0,?,0}, 0b1, (ins nImmSplatI16:$imm)> {
1739  let Inst{9} = imm{9};
1740}
1741def MVE_VMVNimmi32 : MVE_mod_imm<"vmvn", "i32", {?,?,?,?}, 0b1, (ins nImmVMOVI32:$imm)> {
1742  let Inst{11-8} = imm{11-8};
1743}
1744} // let isReMaterializable = 1
1745
1746let Predicates = [HasMVEInt] in {
1747  def : Pat<(v16i8 (ARMvmovImm timm:$simm)),
1748            (v16i8 (MVE_VMOVimmi8  nImmSplatI8:$simm))>;
1749  def : Pat<(v8i16 (ARMvmovImm timm:$simm)),
1750            (v8i16 (MVE_VMOVimmi16 nImmSplatI16:$simm))>;
1751  def : Pat<(v4i32 (ARMvmovImm timm:$simm)),
1752            (v4i32 (MVE_VMOVimmi32 nImmVMOVI32:$simm))>;
1753
1754  def : Pat<(v8i16 (ARMvmvnImm timm:$simm)),
1755            (v8i16 (MVE_VMVNimmi16 nImmSplatI16:$simm))>;
1756  def : Pat<(v4i32 (ARMvmvnImm timm:$simm)),
1757            (v4i32 (MVE_VMVNimmi32 nImmVMOVI32:$simm))>;
1758
1759  def : Pat<(v4f32 (ARMvmovFPImm timm:$simm)),
1760            (v4f32 (MVE_VMOVimmf32 nImmVMOVF32:$simm))>;
1761}
1762
1763class MVE_VMINMAXA<string iname, string suffix, bits<2> size,
1764                   bit bit_12, list<dag> pattern=[]>
1765  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
1766          NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",
1767          pattern> {
1768  bits<4> Qd;
1769  bits<4> Qm;
1770
1771  let Inst{28} = 0b0;
1772  let Inst{25-23} = 0b100;
1773  let Inst{22} = Qd{3};
1774  let Inst{21-20} = 0b11;
1775  let Inst{19-18} = size;
1776  let Inst{17-16} = 0b11;
1777  let Inst{15-13} = Qd{2-0};
1778  let Inst{12} = bit_12;
1779  let Inst{11-6} = 0b111010;
1780  let Inst{5} = Qm{3};
1781  let Inst{4} = 0b0;
1782  let Inst{3-1} = Qm{2-0};
1783  let Inst{0} = 0b1;
1784}
1785
1786def MVE_VMAXAs8  : MVE_VMINMAXA<"vmaxa", "s8",  0b00, 0b0>;
1787def MVE_VMAXAs16 : MVE_VMINMAXA<"vmaxa", "s16", 0b01, 0b0>;
1788def MVE_VMAXAs32 : MVE_VMINMAXA<"vmaxa", "s32", 0b10, 0b0>;
1789
1790def MVE_VMINAs8  : MVE_VMINMAXA<"vmina", "s8",  0b00, 0b1>;
1791def MVE_VMINAs16 : MVE_VMINMAXA<"vmina", "s16", 0b01, 0b1>;
1792def MVE_VMINAs32 : MVE_VMINMAXA<"vmina", "s32", 0b10, 0b1>;
1793
1794// end of MVE Integer instructions
1795
1796// start of mve_imm_shift instructions
1797
1798def MVE_VSHLC : MVE_p<(outs rGPR:$RdmDest, MQPR:$Qd),
1799                      (ins MQPR:$QdSrc, rGPR:$RdmSrc, long_shift:$imm),
1800                      NoItinerary, "vshlc", "", "$QdSrc, $RdmSrc, $imm",
1801                      vpred_n, "$RdmDest = $RdmSrc,$Qd = $QdSrc"> {
1802  bits<5> imm;
1803  bits<4> Qd;
1804  bits<4> RdmDest;
1805
1806  let Inst{28} = 0b0;
1807  let Inst{25-23} = 0b101;
1808  let Inst{22} = Qd{3};
1809  let Inst{21} = 0b1;
1810  let Inst{20-16} = imm{4-0};
1811  let Inst{15-13} = Qd{2-0};
1812  let Inst{12-4} = 0b011111100;
1813  let Inst{3-0} = RdmDest{3-0};
1814}
1815
1816class MVE_shift_imm<dag oops, dag iops, string iname, string suffix,
1817                    string ops, vpred_ops vpred, string cstr,
1818                    list<dag> pattern=[]>
1819  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
1820  bits<4> Qd;
1821  bits<4> Qm;
1822
1823  let Inst{22} = Qd{3};
1824  let Inst{15-13} = Qd{2-0};
1825  let Inst{5} = Qm{3};
1826  let Inst{3-1} = Qm{2-0};
1827}
1828
1829class MVE_VMOVL<string iname, string suffix, bits<2> sz, bit U,
1830              list<dag> pattern=[]>
1831  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),
1832                  iname, suffix, "$Qd, $Qm", vpred_r, "",
1833                  pattern> {
1834  let Inst{28} = U;
1835  let Inst{25-23} = 0b101;
1836  let Inst{21} = 0b1;
1837  let Inst{20-19} = sz{1-0};
1838  let Inst{18-16} = 0b000;
1839  let Inst{11-6} = 0b111101;
1840  let Inst{4} = 0b0;
1841  let Inst{0} = 0b0;
1842}
1843
1844multiclass MVE_VMOVL_shift_half<string iname, string suffix, bits<2> sz, bit U,
1845                                list<dag> pattern=[]> {
1846  def bh : MVE_VMOVL<!strconcat(iname, "b"), suffix, sz, U, pattern> {
1847    let Inst{12} = 0b0;
1848  }
1849  def th : MVE_VMOVL<!strconcat(iname, "t"), suffix, sz, U, pattern> {
1850    let Inst{12} = 0b1;
1851  }
1852}
1853
1854defm MVE_VMOVLs8 : MVE_VMOVL_shift_half<"vmovl", "s8", 0b01, 0b0>;
1855defm MVE_VMOVLu8 : MVE_VMOVL_shift_half<"vmovl", "u8", 0b01, 0b1>;
1856defm MVE_VMOVLs16 : MVE_VMOVL_shift_half<"vmovl", "s16", 0b10, 0b0>;
1857defm MVE_VMOVLu16 : MVE_VMOVL_shift_half<"vmovl", "u16", 0b10, 0b1>;
1858
1859let Predicates = [HasMVEInt] in {
1860  def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i16),
1861            (MVE_VMOVLs16bh MQPR:$src)>;
1862  def : Pat<(sext_inreg (v8i16 MQPR:$src), v8i8),
1863            (MVE_VMOVLs8bh MQPR:$src)>;
1864  def : Pat<(sext_inreg (v4i32 MQPR:$src), v4i8),
1865            (MVE_VMOVLs16bh (MVE_VMOVLs8bh MQPR:$src))>;
1866
1867  // zext_inreg 16 -> 32
1868  def : Pat<(and (v4i32 MQPR:$src), (v4i32 (ARMvmovImm (i32 0xCFF)))),
1869            (MVE_VMOVLu16bh MQPR:$src)>;
1870  // zext_inreg 8 -> 16
1871  def : Pat<(and (v8i16 MQPR:$src), (v8i16 (ARMvmovImm (i32 0x8FF)))),
1872            (MVE_VMOVLu8bh MQPR:$src)>;
1873}
1874
1875
1876class MVE_VSHLL_imm<string iname, string suffix, bit U, bit th,
1877                    dag immops, list<dag> pattern=[]>
1878  : MVE_shift_imm<(outs MQPR:$Qd), !con((ins MQPR:$Qm), immops),
1879                  iname, suffix, "$Qd, $Qm, $imm", vpred_r, "", pattern> {
1880  let Inst{28} = U;
1881  let Inst{25-23} = 0b101;
1882  let Inst{21} = 0b1;
1883  let Inst{12} = th;
1884  let Inst{11-6} = 0b111101;
1885  let Inst{4} = 0b0;
1886  let Inst{0} = 0b0;
1887}
1888
1889// The immediate VSHLL instructions accept shift counts from 1 up to
1890// the lane width (8 or 16), but the full-width shifts have an
1891// entirely separate encoding, given below with 'lw' in the name.
1892
1893class MVE_VSHLL_imm8<string iname, string suffix,
1894                     bit U, bit th, list<dag> pattern=[]>
1895  : MVE_VSHLL_imm<iname, suffix, U, th, (ins mve_shift_imm1_7:$imm), pattern> {
1896  bits<3> imm;
1897  let Inst{20-19} = 0b01;
1898  let Inst{18-16} = imm;
1899}
1900
1901class MVE_VSHLL_imm16<string iname, string suffix,
1902                      bit U, bit th, list<dag> pattern=[]>
1903  : MVE_VSHLL_imm<iname, suffix, U, th, (ins mve_shift_imm1_15:$imm), pattern> {
1904  bits<4> imm;
1905  let Inst{20} = 0b1;
1906  let Inst{19-16} = imm;
1907}
1908
1909def MVE_VSHLL_imms8bh  : MVE_VSHLL_imm8 <"vshllb", "s8", 0b0, 0b0>;
1910def MVE_VSHLL_imms8th  : MVE_VSHLL_imm8 <"vshllt", "s8", 0b0, 0b1>;
1911def MVE_VSHLL_immu8bh  : MVE_VSHLL_imm8 <"vshllb", "u8", 0b1, 0b0>;
1912def MVE_VSHLL_immu8th  : MVE_VSHLL_imm8 <"vshllt", "u8", 0b1, 0b1>;
1913def MVE_VSHLL_imms16bh : MVE_VSHLL_imm16<"vshllb", "s16", 0b0, 0b0>;
1914def MVE_VSHLL_imms16th : MVE_VSHLL_imm16<"vshllt", "s16", 0b0, 0b1>;
1915def MVE_VSHLL_immu16bh : MVE_VSHLL_imm16<"vshllb", "u16", 0b1, 0b0>;
1916def MVE_VSHLL_immu16th : MVE_VSHLL_imm16<"vshllt", "u16", 0b1, 0b1>;
1917
1918class MVE_VSHLL_by_lane_width<string iname, string suffix, bits<2> size,
1919                              bit U, string ops, list<dag> pattern=[]>
1920  : MVE_shift_imm<(outs MQPR:$Qd), (ins MQPR:$Qm),
1921                  iname, suffix, ops, vpred_r, "", pattern> {
1922  let Inst{28} = U;
1923  let Inst{25-23} = 0b100;
1924  let Inst{21-20} = 0b11;
1925  let Inst{19-18} = size{1-0};
1926  let Inst{17-16} = 0b01;
1927  let Inst{11-6} = 0b111000;
1928  let Inst{4} = 0b0;
1929  let Inst{0} = 0b1;
1930}
1931
1932multiclass MVE_VSHLL_lw<string iname, string suffix, bits<2> sz, bit U,
1933                              string ops, list<dag> pattern=[]> {
1934  def bh : MVE_VSHLL_by_lane_width<iname#"b", suffix, sz, U, ops, pattern> {
1935    let Inst{12} = 0b0;
1936  }
1937  def th : MVE_VSHLL_by_lane_width<iname#"t", suffix, sz, U, ops, pattern> {
1938    let Inst{12} = 0b1;
1939  }
1940}
1941
1942defm MVE_VSHLL_lws8  : MVE_VSHLL_lw<"vshll", "s8",  0b00, 0b0, "$Qd, $Qm, #8">;
1943defm MVE_VSHLL_lws16 : MVE_VSHLL_lw<"vshll", "s16", 0b01, 0b0, "$Qd, $Qm, #16">;
1944defm MVE_VSHLL_lwu8  : MVE_VSHLL_lw<"vshll", "u8",  0b00, 0b1, "$Qd, $Qm, #8">;
1945defm MVE_VSHLL_lwu16 : MVE_VSHLL_lw<"vshll", "u16", 0b01, 0b1, "$Qd, $Qm, #16">;
1946
1947class MVE_VxSHRN<string iname, string suffix, bit bit_12, bit bit_28,
1948               dag immops, list<dag> pattern=[]>
1949  : MVE_shift_imm<(outs MQPR:$Qd), !con((ins MQPR:$QdSrc, MQPR:$Qm), immops),
1950                  iname, suffix, "$Qd, $Qm, $imm", vpred_n, "$Qd = $QdSrc",
1951                  pattern> {
1952  bits<5> imm;
1953
1954  let Inst{28} = bit_28;
1955  let Inst{25-23} = 0b101;
1956  let Inst{21} = 0b0;
1957  let Inst{20-16} = imm{4-0};
1958  let Inst{12} = bit_12;
1959  let Inst{11-6} = 0b111111;
1960  let Inst{4} = 0b0;
1961  let Inst{0} = 0b1;
1962}
1963
1964def MVE_VRSHRNi16bh : MVE_VxSHRN<
1965    "vrshrnb", "i16", 0b0, 0b1, (ins shr_imm8:$imm)> {
1966  let Inst{20-19} = 0b01;
1967}
1968def MVE_VRSHRNi16th : MVE_VxSHRN<
1969    "vrshrnt", "i16", 0b1, 0b1,(ins shr_imm8:$imm)> {
1970  let Inst{20-19} = 0b01;
1971}
1972def MVE_VRSHRNi32bh : MVE_VxSHRN<
1973    "vrshrnb", "i32", 0b0, 0b1, (ins shr_imm16:$imm)> {
1974  let Inst{20} = 0b1;
1975}
1976def MVE_VRSHRNi32th : MVE_VxSHRN<
1977    "vrshrnt", "i32", 0b1, 0b1, (ins shr_imm16:$imm)> {
1978  let Inst{20} = 0b1;
1979}
1980
1981def MVE_VSHRNi16bh : MVE_VxSHRN<
1982    "vshrnb", "i16", 0b0, 0b0, (ins shr_imm8:$imm)> {
1983  let Inst{20-19} = 0b01;
1984}
1985def MVE_VSHRNi16th : MVE_VxSHRN<
1986    "vshrnt", "i16", 0b1, 0b0, (ins shr_imm8:$imm)> {
1987  let Inst{20-19} = 0b01;
1988}
1989def MVE_VSHRNi32bh : MVE_VxSHRN<
1990    "vshrnb", "i32", 0b0, 0b0, (ins shr_imm16:$imm)> {
1991  let Inst{20} = 0b1;
1992}
1993def MVE_VSHRNi32th : MVE_VxSHRN<
1994    "vshrnt", "i32", 0b1, 0b0, (ins shr_imm16:$imm)> {
1995  let Inst{20} = 0b1;
1996}
1997
1998class MVE_VxQRSHRUN<string iname, string suffix, bit bit_28, bit bit_12, dag immops,
1999                 list<dag> pattern=[]>
2000  : MVE_shift_imm<(outs MQPR:$Qd), !con((ins MQPR:$QdSrc, MQPR:$Qm), immops),
2001                  iname, suffix, "$Qd, $Qm, $imm", vpred_n, "$Qd = $QdSrc",
2002                  pattern> {
2003  bits<5> imm;
2004
2005  let Inst{28} = bit_28;
2006  let Inst{25-23} = 0b101;
2007  let Inst{21} = 0b0;
2008  let Inst{20-16} = imm{4-0};
2009  let Inst{12} = bit_12;
2010  let Inst{11-6} = 0b111111;
2011  let Inst{4} = 0b0;
2012  let Inst{0} = 0b0;
2013}
2014
2015def MVE_VQRSHRUNs16bh : MVE_VxQRSHRUN<
2016    "vqrshrunb", "s16", 0b1, 0b0, (ins shr_imm8:$imm)> {
2017  let Inst{20-19} = 0b01;
2018}
2019def MVE_VQRSHRUNs16th : MVE_VxQRSHRUN<
2020    "vqrshrunt", "s16", 0b1, 0b1, (ins shr_imm8:$imm)> {
2021  let Inst{20-19} = 0b01;
2022}
2023def MVE_VQRSHRUNs32bh : MVE_VxQRSHRUN<
2024    "vqrshrunb", "s32", 0b1, 0b0, (ins shr_imm16:$imm)> {
2025  let Inst{20} = 0b1;
2026}
2027def MVE_VQRSHRUNs32th : MVE_VxQRSHRUN<
2028    "vqrshrunt", "s32", 0b1, 0b1, (ins shr_imm16:$imm)> {
2029  let Inst{20} = 0b1;
2030}
2031
2032def MVE_VQSHRUNs16bh : MVE_VxQRSHRUN<
2033    "vqshrunb", "s16", 0b0, 0b0, (ins shr_imm8:$imm)> {
2034  let Inst{20-19} = 0b01;
2035}
2036def MVE_VQSHRUNs16th : MVE_VxQRSHRUN<
2037    "vqshrunt", "s16", 0b0, 0b1, (ins shr_imm8:$imm)> {
2038  let Inst{20-19} = 0b01;
2039}
2040def MVE_VQSHRUNs32bh : MVE_VxQRSHRUN<
2041    "vqshrunb", "s32", 0b0, 0b0, (ins shr_imm16:$imm)> {
2042  let Inst{20} = 0b1;
2043}
2044def MVE_VQSHRUNs32th : MVE_VxQRSHRUN<
2045    "vqshrunt", "s32", 0b0, 0b1, (ins shr_imm16:$imm)> {
2046  let Inst{20} = 0b1;
2047}
2048
2049class MVE_VxQRSHRN<string iname, string suffix, bit bit_0, bit bit_12,
2050                   dag immops, list<dag> pattern=[]>
2051  : MVE_shift_imm<(outs MQPR:$Qd), !con((ins MQPR:$QdSrc, MQPR:$Qm), immops),
2052                  iname, suffix, "$Qd, $Qm, $imm", vpred_n, "$Qd = $QdSrc",
2053                  pattern> {
2054  bits<5> imm;
2055
2056  let Inst{25-23} = 0b101;
2057  let Inst{21} = 0b0;
2058  let Inst{20-16} = imm{4-0};
2059  let Inst{12} = bit_12;
2060  let Inst{11-6} = 0b111101;
2061  let Inst{4} = 0b0;
2062  let Inst{0} = bit_0;
2063}
2064
2065multiclass MVE_VxQRSHRN_types<string iname, bit bit_0, bit bit_12> {
2066  def s16 : MVE_VxQRSHRN<iname, "s16", bit_0, bit_12, (ins shr_imm8:$imm)> {
2067    let Inst{28} = 0b0;
2068    let Inst{20-19} = 0b01;
2069  }
2070  def u16 : MVE_VxQRSHRN<iname, "u16", bit_0, bit_12, (ins shr_imm8:$imm)> {
2071    let Inst{28} = 0b1;
2072    let Inst{20-19} = 0b01;
2073  }
2074  def s32 : MVE_VxQRSHRN<iname, "s32", bit_0, bit_12, (ins shr_imm16:$imm)> {
2075    let Inst{28} = 0b0;
2076    let Inst{20} = 0b1;
2077  }
2078  def u32 : MVE_VxQRSHRN<iname, "u32", bit_0, bit_12, (ins shr_imm16:$imm)> {
2079    let Inst{28} = 0b1;
2080    let Inst{20} = 0b1;
2081  }
2082}
2083
2084defm MVE_VQRSHRNbh : MVE_VxQRSHRN_types<"vqrshrnb", 0b1, 0b0>;
2085defm MVE_VQRSHRNth : MVE_VxQRSHRN_types<"vqrshrnt", 0b1, 0b1>;
2086defm MVE_VQSHRNbh  : MVE_VxQRSHRN_types<"vqshrnb", 0b0, 0b0>;
2087defm MVE_VQSHRNth  : MVE_VxQRSHRN_types<"vqshrnt", 0b0, 0b1>;
2088
2089// end of mve_imm_shift instructions
2090
2091// start of mve_shift instructions
2092
2093class MVE_shift_by_vec<string iname, string suffix, bit U,
2094                       bits<2> size, bit bit_4, bit bit_8>
2095  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qm, MQPR:$Qn), NoItinerary,
2096           iname, suffix, "$Qd, $Qm, $Qn", vpred_r, "", []> {
2097  // Shift instructions which take a vector of shift counts
2098  bits<4> Qd;
2099  bits<4> Qm;
2100  bits<4> Qn;
2101
2102  let Inst{28} = U;
2103  let Inst{25-24} = 0b11;
2104  let Inst{23} = 0b0;
2105  let Inst{22} = Qd{3};
2106  let Inst{21-20} = size;
2107  let Inst{19-17} = Qn{2-0};
2108  let Inst{16} = 0b0;
2109  let Inst{15-13} = Qd{2-0};
2110  let Inst{12-9} = 0b0010;
2111  let Inst{8} = bit_8;
2112  let Inst{7} = Qn{3};
2113  let Inst{6} = 0b1;
2114  let Inst{5} = Qm{3};
2115  let Inst{4} = bit_4;
2116  let Inst{3-1} = Qm{2-0};
2117  let Inst{0} = 0b0;
2118}
2119
2120multiclass mve_shift_by_vec_multi<string iname, bit bit_4, bit bit_8> {
2121  def s8  : MVE_shift_by_vec<iname, "s8", 0b0, 0b00, bit_4, bit_8>;
2122  def s16 : MVE_shift_by_vec<iname, "s16", 0b0, 0b01, bit_4, bit_8>;
2123  def s32 : MVE_shift_by_vec<iname, "s32", 0b0, 0b10, bit_4, bit_8>;
2124  def u8  : MVE_shift_by_vec<iname, "u8", 0b1, 0b00, bit_4, bit_8>;
2125  def u16 : MVE_shift_by_vec<iname, "u16", 0b1, 0b01, bit_4, bit_8>;
2126  def u32 : MVE_shift_by_vec<iname, "u32", 0b1, 0b10, bit_4, bit_8>;
2127}
2128
2129defm MVE_VSHL_by_vec   : mve_shift_by_vec_multi<"vshl",   0b0, 0b0>;
2130defm MVE_VQSHL_by_vec  : mve_shift_by_vec_multi<"vqshl",  0b1, 0b0>;
2131defm MVE_VQRSHL_by_vec : mve_shift_by_vec_multi<"vqrshl", 0b1, 0b1>;
2132defm MVE_VRSHL_by_vec  : mve_shift_by_vec_multi<"vrshl",  0b0, 0b1>;
2133
2134let Predicates = [HasMVEInt] in {
2135  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
2136            (v4i32 (MVE_VSHL_by_vecu32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
2137  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
2138            (v8i16 (MVE_VSHL_by_vecu16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
2139  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
2140            (v16i8 (MVE_VSHL_by_vecu8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
2141
2142  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn))),
2143            (v4i32 (MVE_VSHL_by_vecs32 (v4i32 MQPR:$Qm), (v4i32 MQPR:$Qn)))>;
2144  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn))),
2145            (v8i16 (MVE_VSHL_by_vecs16 (v8i16 MQPR:$Qm), (v8i16 MQPR:$Qn)))>;
2146  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn))),
2147            (v16i8 (MVE_VSHL_by_vecs8 (v16i8 MQPR:$Qm), (v16i8 MQPR:$Qn)))>;
2148}
2149
2150class MVE_shift_with_imm<string iname, string suffix, dag oops, dag iops,
2151                         string ops, vpred_ops vpred, string cstr,
2152                         list<dag> pattern=[]>
2153  : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
2154  bits<4> Qd;
2155  bits<4> Qm;
2156
2157  let Inst{23} = 0b1;
2158  let Inst{22} = Qd{3};
2159  let Inst{15-13} = Qd{2-0};
2160  let Inst{12-11} = 0b00;
2161  let Inst{7-6} = 0b01;
2162  let Inst{5} = Qm{3};
2163  let Inst{4} = 0b1;
2164  let Inst{3-1} = Qm{2-0};
2165  let Inst{0} = 0b0;
2166}
2167
2168class MVE_VSxI_imm<string iname, string suffix, bit bit_8, dag imm>
2169  : MVE_shift_with_imm<iname, suffix, (outs MQPR:$Qd),
2170                       !con((ins MQPR:$Qd_src, MQPR:$Qm), imm),
2171                       "$Qd, $Qm, $imm", vpred_n, "$Qd = $Qd_src"> {
2172  bits<6> imm;
2173  let Inst{28} = 0b1;
2174  let Inst{25-24} = 0b11;
2175  let Inst{21-16} = imm;
2176  let Inst{10-9} = 0b10;
2177  let Inst{8} = bit_8;
2178}
2179
2180def MVE_VSRIimm8 : MVE_VSxI_imm<"vsri", "8", 0b0, (ins shr_imm8:$imm)> {
2181  let Inst{21-19} = 0b001;
2182}
2183
2184def MVE_VSRIimm16 : MVE_VSxI_imm<"vsri", "16", 0b0, (ins shr_imm16:$imm)> {
2185  let Inst{21-20} = 0b01;
2186}
2187
2188def MVE_VSRIimm32 : MVE_VSxI_imm<"vsri", "32", 0b0, (ins shr_imm32:$imm)> {
2189  let Inst{21} = 0b1;
2190}
2191
2192def MVE_VSLIimm8 : MVE_VSxI_imm<"vsli", "8", 0b1, (ins imm0_7:$imm)> {
2193  let Inst{21-19} = 0b001;
2194}
2195
2196def MVE_VSLIimm16 : MVE_VSxI_imm<"vsli", "16", 0b1, (ins imm0_15:$imm)> {
2197  let Inst{21-20} = 0b01;
2198}
2199
2200def MVE_VSLIimm32 : MVE_VSxI_imm<"vsli", "32", 0b1,(ins imm0_31:$imm)> {
2201  let Inst{21} = 0b1;
2202}
2203
2204class MVE_VQSHL_imm<string suffix, dag imm>
2205  : MVE_shift_with_imm<"vqshl", suffix, (outs MQPR:$Qd),
2206                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
2207                       vpred_r, ""> {
2208  bits<6> imm;
2209
2210  let Inst{25-24} = 0b11;
2211  let Inst{21-16} = imm;
2212  let Inst{10-8} = 0b111;
2213}
2214
2215def MVE_VSLIimms8 : MVE_VQSHL_imm<"s8", (ins imm0_7:$imm)> {
2216  let Inst{28} = 0b0;
2217  let Inst{21-19} = 0b001;
2218}
2219
2220def MVE_VSLIimmu8 : MVE_VQSHL_imm<"u8", (ins imm0_7:$imm)> {
2221  let Inst{28} = 0b1;
2222  let Inst{21-19} = 0b001;
2223}
2224
2225def MVE_VSLIimms16 : MVE_VQSHL_imm<"s16", (ins imm0_15:$imm)> {
2226  let Inst{28} = 0b0;
2227  let Inst{21-20} = 0b01;
2228}
2229
2230def MVE_VSLIimmu16 : MVE_VQSHL_imm<"u16", (ins imm0_15:$imm)> {
2231  let Inst{28} = 0b1;
2232  let Inst{21-20} = 0b01;
2233}
2234
2235def MVE_VSLIimms32 : MVE_VQSHL_imm<"s32", (ins imm0_31:$imm)> {
2236  let Inst{28} = 0b0;
2237  let Inst{21} = 0b1;
2238}
2239
2240def MVE_VSLIimmu32 : MVE_VQSHL_imm<"u32", (ins imm0_31:$imm)> {
2241  let Inst{28} = 0b1;
2242  let Inst{21} = 0b1;
2243}
2244
2245class MVE_VQSHLU_imm<string suffix, dag imm>
2246  : MVE_shift_with_imm<"vqshlu", suffix, (outs MQPR:$Qd),
2247                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
2248                       vpred_r, ""> {
2249  bits<6> imm;
2250
2251  let Inst{28} = 0b1;
2252  let Inst{25-24} = 0b11;
2253  let Inst{21-16} = imm;
2254  let Inst{10-8} = 0b110;
2255}
2256
2257def MVE_VQSHLU_imms8 : MVE_VQSHLU_imm<"s8", (ins imm0_7:$imm)> {
2258  let Inst{21-19} = 0b001;
2259}
2260
2261def MVE_VQSHLU_imms16 : MVE_VQSHLU_imm<"s16", (ins imm0_15:$imm)> {
2262  let Inst{21-20} = 0b01;
2263}
2264
2265def MVE_VQSHLU_imms32 : MVE_VQSHLU_imm<"s32", (ins imm0_31:$imm)> {
2266  let Inst{21} = 0b1;
2267}
2268
2269class MVE_VRSHR_imm<string suffix, dag imm>
2270  : MVE_shift_with_imm<"vrshr", suffix, (outs MQPR:$Qd),
2271                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
2272                       vpred_r, ""> {
2273  bits<6> imm;
2274
2275  let Inst{25-24} = 0b11;
2276  let Inst{21-16} = imm;
2277  let Inst{10-8} = 0b010;
2278}
2279
2280def MVE_VRSHR_imms8 : MVE_VRSHR_imm<"s8", (ins shr_imm8:$imm)> {
2281  let Inst{28} = 0b0;
2282  let Inst{21-19} = 0b001;
2283}
2284
2285def MVE_VRSHR_immu8 : MVE_VRSHR_imm<"u8", (ins shr_imm8:$imm)> {
2286  let Inst{28} = 0b1;
2287  let Inst{21-19} = 0b001;
2288}
2289
2290def MVE_VRSHR_imms16 : MVE_VRSHR_imm<"s16", (ins shr_imm16:$imm)> {
2291  let Inst{28} = 0b0;
2292  let Inst{21-20} = 0b01;
2293}
2294
2295def MVE_VRSHR_immu16 : MVE_VRSHR_imm<"u16", (ins shr_imm16:$imm)> {
2296  let Inst{28} = 0b1;
2297  let Inst{21-20} = 0b01;
2298}
2299
2300def MVE_VRSHR_imms32 : MVE_VRSHR_imm<"s32", (ins shr_imm32:$imm)> {
2301  let Inst{28} = 0b0;
2302  let Inst{21} = 0b1;
2303}
2304
2305def MVE_VRSHR_immu32 : MVE_VRSHR_imm<"u32", (ins shr_imm32:$imm)> {
2306  let Inst{28} = 0b1;
2307  let Inst{21} = 0b1;
2308}
2309
2310class MVE_VSHR_imm<string suffix, dag imm>
2311  : MVE_shift_with_imm<"vshr", suffix, (outs MQPR:$Qd),
2312                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
2313                       vpred_r, ""> {
2314  bits<6> imm;
2315
2316  let Inst{25-24} = 0b11;
2317  let Inst{21-16} = imm;
2318  let Inst{10-8} = 0b000;
2319}
2320
2321def MVE_VSHR_imms8 : MVE_VSHR_imm<"s8", (ins shr_imm8:$imm)> {
2322  let Inst{28} = 0b0;
2323  let Inst{21-19} = 0b001;
2324}
2325
2326def MVE_VSHR_immu8 : MVE_VSHR_imm<"u8", (ins shr_imm8:$imm)> {
2327  let Inst{28} = 0b1;
2328  let Inst{21-19} = 0b001;
2329}
2330
2331def MVE_VSHR_imms16 : MVE_VSHR_imm<"s16", (ins shr_imm16:$imm)> {
2332  let Inst{28} = 0b0;
2333  let Inst{21-20} = 0b01;
2334}
2335
2336def MVE_VSHR_immu16 : MVE_VSHR_imm<"u16", (ins shr_imm16:$imm)> {
2337  let Inst{28} = 0b1;
2338  let Inst{21-20} = 0b01;
2339}
2340
2341def MVE_VSHR_imms32 : MVE_VSHR_imm<"s32", (ins shr_imm32:$imm)> {
2342  let Inst{28} = 0b0;
2343  let Inst{21} = 0b1;
2344}
2345
2346def MVE_VSHR_immu32 : MVE_VSHR_imm<"u32", (ins shr_imm32:$imm)> {
2347  let Inst{28} = 0b1;
2348  let Inst{21} = 0b1;
2349}
2350
2351class MVE_VSHL_imm<string suffix, dag imm>
2352  : MVE_shift_with_imm<"vshl", suffix, (outs MQPR:$Qd),
2353                       !con((ins MQPR:$Qm), imm), "$Qd, $Qm, $imm",
2354                       vpred_r, ""> {
2355  bits<6> imm;
2356
2357  let Inst{28} = 0b0;
2358  let Inst{25-24} = 0b11;
2359  let Inst{21-16} = imm;
2360  let Inst{10-8} = 0b101;
2361}
2362
2363def MVE_VSHL_immi8 : MVE_VSHL_imm<"i8", (ins imm0_7:$imm)> {
2364  let Inst{21-19} = 0b001;
2365}
2366
2367def MVE_VSHL_immi16 : MVE_VSHL_imm<"i16", (ins imm0_15:$imm)> {
2368  let Inst{21-20} = 0b01;
2369}
2370
2371def MVE_VSHL_immi32 : MVE_VSHL_imm<"i32", (ins imm0_31:$imm)> {
2372  let Inst{21} = 0b1;
2373}
2374
2375let Predicates = [HasMVEInt] in {
2376  def : Pat<(v4i32 (ARMvshlImm (v4i32 MQPR:$src), imm0_31:$imm)),
2377            (v4i32 (MVE_VSHL_immi32 (v4i32 MQPR:$src), imm0_31:$imm))>;
2378  def : Pat<(v8i16 (ARMvshlImm (v8i16 MQPR:$src), imm0_15:$imm)),
2379            (v8i16 (MVE_VSHL_immi16 (v8i16 MQPR:$src), imm0_15:$imm))>;
2380  def : Pat<(v16i8 (ARMvshlImm (v16i8 MQPR:$src), imm0_7:$imm)),
2381            (v16i8 (MVE_VSHL_immi8 (v16i8 MQPR:$src), imm0_7:$imm))>;
2382
2383  def : Pat<(v4i32 (ARMvshruImm (v4i32 MQPR:$src), imm0_31:$imm)),
2384            (v4i32 (MVE_VSHR_immu32 (v4i32 MQPR:$src), imm0_31:$imm))>;
2385  def : Pat<(v8i16 (ARMvshruImm (v8i16 MQPR:$src), imm0_15:$imm)),
2386            (v8i16 (MVE_VSHR_immu16 (v8i16 MQPR:$src), imm0_15:$imm))>;
2387  def : Pat<(v16i8 (ARMvshruImm (v16i8 MQPR:$src), imm0_7:$imm)),
2388            (v16i8 (MVE_VSHR_immu8 (v16i8 MQPR:$src), imm0_7:$imm))>;
2389
2390  def : Pat<(v4i32 (ARMvshrsImm (v4i32 MQPR:$src), imm0_31:$imm)),
2391            (v4i32 (MVE_VSHR_imms32 (v4i32 MQPR:$src), imm0_31:$imm))>;
2392  def : Pat<(v8i16 (ARMvshrsImm (v8i16 MQPR:$src), imm0_15:$imm)),
2393            (v8i16 (MVE_VSHR_imms16 (v8i16 MQPR:$src), imm0_15:$imm))>;
2394  def : Pat<(v16i8 (ARMvshrsImm (v16i8 MQPR:$src), imm0_7:$imm)),
2395            (v16i8 (MVE_VSHR_imms8 (v16i8 MQPR:$src), imm0_7:$imm))>;
2396}
2397
2398// end of mve_shift instructions
2399
2400// start of MVE Floating Point instructions
2401
2402class MVE_float<string iname, string suffix, dag oops, dag iops, string ops,
2403                vpred_ops vpred, string cstr, list<dag> pattern=[]>
2404  : MVE_f<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
2405  bits<4> Qm;
2406
2407  let Inst{12} = 0b0;
2408  let Inst{6} = 0b1;
2409  let Inst{5} = Qm{3};
2410  let Inst{3-1} = Qm{2-0};
2411  let Inst{0} = 0b0;
2412}
2413
2414class MVE_VRINT<string rmode, bits<3> op, string suffix, bits<2> size,
2415                list<dag> pattern=[]>
2416  : MVE_float<!strconcat("vrint", rmode), suffix, (outs MQPR:$Qd),
2417              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
2418  bits<4> Qd;
2419
2420  let Inst{28} = 0b1;
2421  let Inst{25-23} = 0b111;
2422  let Inst{22} = Qd{3};
2423  let Inst{21-20} = 0b11;
2424  let Inst{19-18} = size;
2425  let Inst{17-16} = 0b10;
2426  let Inst{15-13} = Qd{2-0};
2427  let Inst{11-10} = 0b01;
2428  let Inst{9-7} = op{2-0};
2429  let Inst{4} = 0b0;
2430
2431}
2432
2433multiclass MVE_VRINT_ops<string suffix, bits<2> size, list<dag> pattern=[]> {
2434  def N : MVE_VRINT<"n", 0b000, suffix, size, pattern>;
2435  def X : MVE_VRINT<"x", 0b001, suffix, size, pattern>;
2436  def A : MVE_VRINT<"a", 0b010, suffix, size, pattern>;
2437  def Z : MVE_VRINT<"z", 0b011, suffix, size, pattern>;
2438  def M : MVE_VRINT<"m", 0b101, suffix, size, pattern>;
2439  def P : MVE_VRINT<"p", 0b111, suffix, size, pattern>;
2440}
2441
2442defm MVE_VRINTf16 : MVE_VRINT_ops<"f16", 0b01>;
2443defm MVE_VRINTf32 : MVE_VRINT_ops<"f32", 0b10>;
2444
2445let Predicates = [HasMVEFloat] in {
2446  def : Pat<(v4f32 (frint (v4f32 MQPR:$val1))),
2447            (v4f32 (MVE_VRINTf32X (v4f32 MQPR:$val1)))>;
2448  def : Pat<(v8f16 (frint (v8f16 MQPR:$val1))),
2449            (v8f16 (MVE_VRINTf16X (v8f16 MQPR:$val1)))>;
2450  def : Pat<(v4f32 (fround (v4f32 MQPR:$val1))),
2451            (v4f32 (MVE_VRINTf32A (v4f32 MQPR:$val1)))>;
2452  def : Pat<(v8f16 (fround (v8f16 MQPR:$val1))),
2453            (v8f16 (MVE_VRINTf16A (v8f16 MQPR:$val1)))>;
2454  def : Pat<(v4f32 (ftrunc (v4f32 MQPR:$val1))),
2455            (v4f32 (MVE_VRINTf32Z (v4f32 MQPR:$val1)))>;
2456  def : Pat<(v8f16 (ftrunc (v8f16 MQPR:$val1))),
2457            (v8f16 (MVE_VRINTf16Z (v8f16 MQPR:$val1)))>;
2458  def : Pat<(v4f32 (ffloor (v4f32 MQPR:$val1))),
2459            (v4f32 (MVE_VRINTf32M (v4f32 MQPR:$val1)))>;
2460  def : Pat<(v8f16 (ffloor (v8f16 MQPR:$val1))),
2461            (v8f16 (MVE_VRINTf16M (v8f16 MQPR:$val1)))>;
2462  def : Pat<(v4f32 (fceil (v4f32 MQPR:$val1))),
2463            (v4f32 (MVE_VRINTf32P (v4f32 MQPR:$val1)))>;
2464  def : Pat<(v8f16 (fceil (v8f16 MQPR:$val1))),
2465            (v8f16 (MVE_VRINTf16P (v8f16 MQPR:$val1)))>;
2466}
2467
2468class MVEFloatArithNeon<string iname, string suffix, bit size,
2469                           dag oops, dag iops, string ops,
2470                           vpred_ops vpred, string cstr, list<dag> pattern=[]>
2471  : MVE_float<iname, suffix, oops, iops, ops, vpred, cstr, pattern> {
2472  let Inst{20} = size;
2473  let Inst{16} = 0b0;
2474}
2475
2476class MVE_VMUL_fp<string suffix, bit size, list<dag> pattern=[]>
2477  : MVEFloatArithNeon<"vmul", suffix, size, (outs MQPR:$Qd),
2478                      (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm", vpred_r, "",
2479                      pattern> {
2480  bits<4> Qd;
2481  bits<4> Qn;
2482
2483  let Inst{28} = 0b1;
2484  let Inst{25-23} = 0b110;
2485  let Inst{22} = Qd{3};
2486  let Inst{21} = 0b0;
2487  let Inst{19-17} = Qn{2-0};
2488  let Inst{15-13} = Qd{2-0};
2489  let Inst{12-8} = 0b01101;
2490  let Inst{7} = Qn{3};
2491  let Inst{4} = 0b1;
2492}
2493
2494def MVE_VMULf32 : MVE_VMUL_fp<"f32", 0b0>;
2495def MVE_VMULf16 : MVE_VMUL_fp<"f16", 0b1>;
2496
2497let Predicates = [HasMVEFloat] in {
2498  def : Pat<(v4f32 (fmul (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
2499            (v4f32 (MVE_VMULf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
2500  def : Pat<(v8f16 (fmul (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
2501            (v8f16 (MVE_VMULf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
2502}
2503
2504class MVE_VCMLA<string suffix, bit size, list<dag> pattern=[]>
2505  : MVEFloatArithNeon<"vcmla", suffix, size, (outs MQPR:$Qd),
2506                         (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
2507                         "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", pattern> {
2508  bits<4> Qd;
2509  bits<4> Qn;
2510  bits<2> rot;
2511
2512  let Inst{28} = 0b1;
2513  let Inst{25} = 0b0;
2514  let Inst{24-23} = rot;
2515  let Inst{22} = Qd{3};
2516  let Inst{21} = 0b1;
2517  let Inst{19-17} = Qn{2-0};
2518  let Inst{15-13} = Qd{2-0};
2519  let Inst{12-8} = 0b01000;
2520  let Inst{7} = Qn{3};
2521  let Inst{4} = 0b0;
2522}
2523
2524def MVE_VCMLAf16 : MVE_VCMLA<"f16", 0b0>;
2525def MVE_VCMLAf32 : MVE_VCMLA<"f32", 0b1>;
2526
2527class MVE_VADDSUBFMA_fp<string iname, string suffix, bit size, bit bit_4,
2528                        bit bit_8, bit bit_21, dag iops=(ins),
2529                        vpred_ops vpred=vpred_r, string cstr="",
2530                        list<dag> pattern=[]>
2531  : MVEFloatArithNeon<iname, suffix, size, (outs MQPR:$Qd),
2532                      !con(iops, (ins MQPR:$Qn, MQPR:$Qm)), "$Qd, $Qn, $Qm",
2533                      vpred, cstr, pattern> {
2534  bits<4> Qd;
2535  bits<4> Qn;
2536
2537  let Inst{28} = 0b0;
2538  let Inst{25-23} = 0b110;
2539  let Inst{22} = Qd{3};
2540  let Inst{21} = bit_21;
2541  let Inst{19-17} = Qn{2-0};
2542  let Inst{15-13} = Qd{2-0};
2543  let Inst{11-9} = 0b110;
2544  let Inst{8} = bit_8;
2545  let Inst{7} = Qn{3};
2546  let Inst{4} = bit_4;
2547}
2548
2549def MVE_VFMAf32 : MVE_VADDSUBFMA_fp<"vfma", "f32", 0b0, 0b1, 0b0, 0b0,
2550    (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
2551def MVE_VFMAf16 : MVE_VADDSUBFMA_fp<"vfma", "f16", 0b1, 0b1, 0b0, 0b0,
2552    (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
2553
2554def MVE_VFMSf32 : MVE_VADDSUBFMA_fp<"vfms", "f32", 0b0, 0b1, 0b0, 0b1,
2555    (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
2556def MVE_VFMSf16 : MVE_VADDSUBFMA_fp<"vfms", "f16", 0b1, 0b1, 0b0, 0b1,
2557    (ins MQPR:$Qd_src), vpred_n, "$Qd = $Qd_src">;
2558
2559def MVE_VADDf32 : MVE_VADDSUBFMA_fp<"vadd", "f32", 0b0, 0b0, 0b1, 0b0>;
2560def MVE_VADDf16 : MVE_VADDSUBFMA_fp<"vadd", "f16", 0b1, 0b0, 0b1, 0b0>;
2561
2562let Predicates = [HasMVEFloat] in {
2563  def : Pat<(v4f32 (fadd (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
2564            (v4f32 (MVE_VADDf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
2565  def : Pat<(v8f16 (fadd (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
2566            (v8f16 (MVE_VADDf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
2567}
2568
2569def MVE_VSUBf32 : MVE_VADDSUBFMA_fp<"vsub", "f32", 0b0, 0b0, 0b1, 0b1>;
2570def MVE_VSUBf16 : MVE_VADDSUBFMA_fp<"vsub", "f16", 0b1, 0b0, 0b1, 0b1>;
2571
2572let Predicates = [HasMVEFloat] in {
2573  def : Pat<(v4f32 (fsub (v4f32 MQPR:$val1), (v4f32 MQPR:$val2))),
2574            (v4f32 (MVE_VSUBf32 (v4f32 MQPR:$val1), (v4f32 MQPR:$val2)))>;
2575  def : Pat<(v8f16 (fsub (v8f16 MQPR:$val1), (v8f16 MQPR:$val2))),
2576            (v8f16 (MVE_VSUBf16 (v8f16 MQPR:$val1), (v8f16 MQPR:$val2)))>;
2577}
2578
2579class MVE_VCADD<string suffix, bit size, list<dag> pattern=[]>
2580  : MVEFloatArithNeon<"vcadd", suffix, size, (outs MQPR:$Qd),
2581                         (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
2582                         "$Qd, $Qn, $Qm, $rot", vpred_r, "", pattern> {
2583  bits<4> Qd;
2584  bits<4> Qn;
2585  bit rot;
2586
2587  let Inst{28} = 0b1;
2588  let Inst{25} = 0b0;
2589  let Inst{24} = rot;
2590  let Inst{23} = 0b1;
2591  let Inst{22} = Qd{3};
2592  let Inst{21} = 0b0;
2593  let Inst{19-17} = Qn{2-0};
2594  let Inst{15-13} = Qd{2-0};
2595  let Inst{12-8} = 0b01000;
2596  let Inst{7} = Qn{3};
2597  let Inst{4} = 0b0;
2598}
2599
2600def MVE_VCADDf16 : MVE_VCADD<"f16", 0b0>;
2601def MVE_VCADDf32 : MVE_VCADD<"f32", 0b1>;
2602
2603class MVE_VABD_fp<string suffix, bit size>
2604  : MVE_float<"vabd", suffix, (outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
2605              "$Qd, $Qn, $Qm", vpred_r, ""> {
2606  bits<4> Qd;
2607  bits<4> Qn;
2608
2609  let Inst{28} = 0b1;
2610  let Inst{25-23} = 0b110;
2611  let Inst{22} = Qd{3};
2612  let Inst{21} = 0b1;
2613  let Inst{20} = size;
2614  let Inst{19-17} = Qn{2-0};
2615  let Inst{16} = 0b0;
2616  let Inst{15-13} = Qd{2-0};
2617  let Inst{11-8} = 0b1101;
2618  let Inst{7} = Qn{3};
2619  let Inst{4} = 0b0;
2620}
2621
2622def MVE_VABDf32 : MVE_VABD_fp<"f32", 0b0>;
2623def MVE_VABDf16 : MVE_VABD_fp<"f16", 0b1>;
2624
2625class MVE_VCVT_fix<string suffix, bit fsi, bit U, bit op,
2626                   Operand imm_operand_type, list<dag> pattern=[]>
2627  : MVE_float<"vcvt", suffix,
2628              (outs MQPR:$Qd), (ins MQPR:$Qm, imm_operand_type:$imm6),
2629              "$Qd, $Qm, $imm6", vpred_r, "", pattern> {
2630  bits<4> Qd;
2631  bits<6> imm6;
2632
2633  let Inst{28} = U;
2634  let Inst{25-23} = 0b111;
2635  let Inst{22} = Qd{3};
2636  let Inst{21} = 0b1;
2637  let Inst{19-16} = imm6{3-0};
2638  let Inst{15-13} = Qd{2-0};
2639  let Inst{11-10} = 0b11;
2640  let Inst{9} = fsi;
2641  let Inst{8} = op;
2642  let Inst{7} = 0b0;
2643  let Inst{4} = 0b1;
2644
2645  let DecoderMethod = "DecodeMVEVCVTt1fp";
2646}
2647
2648class MVE_VCVT_imm_asmop<int Bits> : AsmOperandClass {
2649  let PredicateMethod = "isImmediate<1," # Bits # ">";
2650  let DiagnosticString =
2651      "MVE fixed-point immediate operand must be between 1 and " # Bits;
2652  let Name = "MVEVcvtImm" # Bits;
2653  let RenderMethod = "addImmOperands";
2654}
2655class MVE_VCVT_imm<int Bits>: Operand<i32> {
2656  let ParserMatchClass = MVE_VCVT_imm_asmop<Bits>;
2657  let EncoderMethod = "getNEONVcvtImm32OpValue";
2658  let DecoderMethod = "DecodeVCVTImmOperand";
2659}
2660
2661class MVE_VCVT_fix_f32<string suffix, bit U, bit op>
2662    : MVE_VCVT_fix<suffix, 0b1, U, op, MVE_VCVT_imm<32>> {
2663  let Inst{20} = imm6{4};
2664}
2665class MVE_VCVT_fix_f16<string suffix, bit U, bit op>
2666    : MVE_VCVT_fix<suffix, 0b0, U, op, MVE_VCVT_imm<16>> {
2667  let Inst{20} = 0b1;
2668}
2669
2670def MVE_VCVTf16s16_fix : MVE_VCVT_fix_f16<"f16.s16", 0b0, 0b0>;
2671def MVE_VCVTs16f16_fix : MVE_VCVT_fix_f16<"s16.f16", 0b0, 0b1>;
2672def MVE_VCVTf16u16_fix : MVE_VCVT_fix_f16<"f16.u16", 0b1, 0b0>;
2673def MVE_VCVTu16f16_fix : MVE_VCVT_fix_f16<"u16.f16", 0b1, 0b1>;
2674def MVE_VCVTf32s32_fix : MVE_VCVT_fix_f32<"f32.s32", 0b0, 0b0>;
2675def MVE_VCVTs32f32_fix : MVE_VCVT_fix_f32<"s32.f32", 0b0, 0b1>;
2676def MVE_VCVTf32u32_fix : MVE_VCVT_fix_f32<"f32.u32", 0b1, 0b0>;
2677def MVE_VCVTu32f32_fix : MVE_VCVT_fix_f32<"u32.f32", 0b1, 0b1>;
2678
2679class MVE_VCVT_fp_int_anpm<string suffix, bits<2> size, bit op, string anpm,
2680                bits<2> rm, list<dag> pattern=[]>
2681  : MVE_float<!strconcat("vcvt", anpm), suffix, (outs MQPR:$Qd),
2682              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
2683  bits<4> Qd;
2684
2685  let Inst{28} = 0b1;
2686  let Inst{25-23} = 0b111;
2687  let Inst{22} = Qd{3};
2688  let Inst{21-20} = 0b11;
2689  let Inst{19-18} = size;
2690  let Inst{17-16} = 0b11;
2691  let Inst{15-13} = Qd{2-0};
2692  let Inst{12-10} = 0b000;
2693  let Inst{9-8} = rm;
2694  let Inst{7} = op;
2695  let Inst{4} = 0b0;
2696}
2697
2698multiclass MVE_VCVT_fp_int_anpm_multi<string suffix, bits<2> size, bit op,
2699                                list<dag> pattern=[]> {
2700  def a : MVE_VCVT_fp_int_anpm<suffix, size, op, "a", 0b00>;
2701  def n : MVE_VCVT_fp_int_anpm<suffix, size, op, "n", 0b01>;
2702  def p : MVE_VCVT_fp_int_anpm<suffix, size, op, "p", 0b10>;
2703  def m : MVE_VCVT_fp_int_anpm<suffix, size, op, "m", 0b11>;
2704}
2705
2706// This defines instructions such as MVE_VCVTu16f16a, with an explicit
2707// rounding-mode suffix on the mnemonic. The class below will define
2708// the bare MVE_VCVTu16f16 (with implied rounding toward zero).
2709defm MVE_VCVTs16f16 : MVE_VCVT_fp_int_anpm_multi<"s16.f16", 0b01, 0b0>;
2710defm MVE_VCVTu16f16 : MVE_VCVT_fp_int_anpm_multi<"u16.f16", 0b01, 0b1>;
2711defm MVE_VCVTs32f32 : MVE_VCVT_fp_int_anpm_multi<"s32.f32", 0b10, 0b0>;
2712defm MVE_VCVTu32f32 : MVE_VCVT_fp_int_anpm_multi<"u32.f32", 0b10, 0b1>;
2713
2714class MVE_VCVT_fp_int<string suffix, bits<2> size, bits<2> op,
2715                      list<dag> pattern=[]>
2716  : MVE_float<"vcvt", suffix, (outs MQPR:$Qd),
2717              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
2718  bits<4> Qd;
2719
2720  let Inst{28} = 0b1;
2721  let Inst{25-23} = 0b111;
2722  let Inst{22} = Qd{3};
2723  let Inst{21-20} = 0b11;
2724  let Inst{19-18} = size;
2725  let Inst{17-16} = 0b11;
2726  let Inst{15-13} = Qd{2-0};
2727  let Inst{12-9} = 0b0011;
2728  let Inst{8-7} = op;
2729  let Inst{4} = 0b0;
2730}
2731
2732// The unsuffixed VCVT for float->int implicitly rounds toward zero,
2733// which I reflect here in the llvm instruction names
2734def MVE_VCVTs16f16z : MVE_VCVT_fp_int<"s16.f16", 0b01, 0b10>;
2735def MVE_VCVTu16f16z : MVE_VCVT_fp_int<"u16.f16", 0b01, 0b11>;
2736def MVE_VCVTs32f32z : MVE_VCVT_fp_int<"s32.f32", 0b10, 0b10>;
2737def MVE_VCVTu32f32z : MVE_VCVT_fp_int<"u32.f32", 0b10, 0b11>;
2738// Whereas VCVT for int->float rounds to nearest
2739def MVE_VCVTf16s16n : MVE_VCVT_fp_int<"f16.s16", 0b01, 0b00>;
2740def MVE_VCVTf16u16n : MVE_VCVT_fp_int<"f16.u16", 0b01, 0b01>;
2741def MVE_VCVTf32s32n : MVE_VCVT_fp_int<"f32.s32", 0b10, 0b00>;
2742def MVE_VCVTf32u32n : MVE_VCVT_fp_int<"f32.u32", 0b10, 0b01>;
2743
2744let Predicates = [HasMVEFloat] in {
2745  def : Pat<(v4i32 (fp_to_sint (v4f32 MQPR:$src))),
2746            (v4i32 (MVE_VCVTs32f32z (v4f32 MQPR:$src)))>;
2747  def : Pat<(v4i32 (fp_to_uint (v4f32 MQPR:$src))),
2748            (v4i32 (MVE_VCVTu32f32z (v4f32 MQPR:$src)))>;
2749  def : Pat<(v8i16 (fp_to_sint (v8f16 MQPR:$src))),
2750            (v8i16 (MVE_VCVTs16f16z (v8f16 MQPR:$src)))>;
2751  def : Pat<(v8i16 (fp_to_uint (v8f16 MQPR:$src))),
2752            (v8i16 (MVE_VCVTu16f16z (v8f16 MQPR:$src)))>;
2753  def : Pat<(v4f32 (sint_to_fp (v4i32 MQPR:$src))),
2754            (v4f32 (MVE_VCVTf32s32n (v4i32 MQPR:$src)))>;
2755  def : Pat<(v4f32 (uint_to_fp (v4i32 MQPR:$src))),
2756            (v4f32 (MVE_VCVTf32u32n (v4i32 MQPR:$src)))>;
2757  def : Pat<(v8f16 (sint_to_fp (v8i16 MQPR:$src))),
2758            (v8f16 (MVE_VCVTf16s16n (v8i16 MQPR:$src)))>;
2759  def : Pat<(v8f16 (uint_to_fp (v8i16 MQPR:$src))),
2760            (v8f16 (MVE_VCVTf16u16n (v8i16 MQPR:$src)))>;
2761}
2762
2763class MVE_VABSNEG_fp<string iname, string suffix, bits<2> size, bit negate,
2764                   list<dag> pattern=[]>
2765  : MVE_float<iname, suffix, (outs MQPR:$Qd),
2766              (ins MQPR:$Qm), "$Qd, $Qm", vpred_r, "", pattern> {
2767  bits<4> Qd;
2768
2769  let Inst{28} = 0b1;
2770  let Inst{25-23} = 0b111;
2771  let Inst{22} = Qd{3};
2772  let Inst{21-20} = 0b11;
2773  let Inst{19-18} = size;
2774  let Inst{17-16} = 0b01;
2775  let Inst{15-13} = Qd{2-0};
2776  let Inst{11-8} = 0b0111;
2777  let Inst{7} = negate;
2778  let Inst{4} = 0b0;
2779}
2780
2781def MVE_VABSf16 : MVE_VABSNEG_fp<"vabs", "f16", 0b01, 0b0>;
2782def MVE_VABSf32 : MVE_VABSNEG_fp<"vabs", "f32", 0b10, 0b0>;
2783
2784let Predicates = [HasMVEFloat] in {
2785  def : Pat<(v8f16 (fabs MQPR:$src)),
2786            (MVE_VABSf16 MQPR:$src)>;
2787  def : Pat<(v4f32 (fabs MQPR:$src)),
2788            (MVE_VABSf32 MQPR:$src)>;
2789}
2790
2791def MVE_VNEGf16 : MVE_VABSNEG_fp<"vneg", "f16", 0b01, 0b1>;
2792def MVE_VNEGf32 : MVE_VABSNEG_fp<"vneg", "f32", 0b10, 0b1>;
2793
2794let Predicates = [HasMVEFloat] in {
2795  def : Pat<(v8f16 (fneg MQPR:$src)),
2796            (MVE_VNEGf16 MQPR:$src)>;
2797  def : Pat<(v4f32 (fneg MQPR:$src)),
2798            (MVE_VNEGf32 MQPR:$src)>;
2799}
2800
2801class MVE_VMAXMINNMA<string iname, string suffix, bit size, bit bit_12,
2802                     list<dag> pattern=[]>
2803  : MVE_f<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
2804          NoItinerary, iname, suffix, "$Qd, $Qm", vpred_n, "$Qd = $Qd_src",
2805          pattern> {
2806  bits<4> Qd;
2807  bits<4> Qm;
2808
2809  let Inst{28} = size;
2810  let Inst{25-23} = 0b100;
2811  let Inst{22} = Qd{3};
2812  let Inst{21-16} = 0b111111;
2813  let Inst{15-13} = Qd{2-0};
2814  let Inst{12} = bit_12;
2815  let Inst{11-6} = 0b111010;
2816  let Inst{5} = Qm{3};
2817  let Inst{4} = 0b0;
2818  let Inst{3-1} = Qm{2-0};
2819  let Inst{0} = 0b1;
2820}
2821
2822def MVE_VMAXNMAf32 : MVE_VMAXMINNMA<"vmaxnma", "f32", 0b0, 0b0>;
2823def MVE_VMAXNMAf16 : MVE_VMAXMINNMA<"vmaxnma", "f16", 0b1, 0b0>;
2824
2825def MVE_VMINNMAf32 : MVE_VMAXMINNMA<"vminnma", "f32", 0b0, 0b1>;
2826def MVE_VMINNMAf16 : MVE_VMAXMINNMA<"vminnma", "f16", 0b1, 0b1>;
2827
2828// end of MVE Floating Point instructions
2829
2830// start of MVE compares
2831
2832class MVE_VCMPqq<string suffix, bit bit_28, bits<2> bits_21_20,
2833                 VCMPPredicateOperand predtype, list<dag> pattern=[]>
2834  : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, MQPR:$Qm, predtype:$fc),
2835           NoItinerary, "vcmp", suffix, "$fc, $Qn, $Qm", vpred_n, "", pattern> {
2836  // Base class for comparing two vector registers
2837  bits<3> fc;
2838  bits<4> Qn;
2839  bits<4> Qm;
2840
2841  let Inst{28} = bit_28;
2842  let Inst{25-22} = 0b1000;
2843  let Inst{21-20} = bits_21_20;
2844  let Inst{19-17} = Qn{2-0};
2845  let Inst{16-13} = 0b1000;
2846  let Inst{12} = fc{2};
2847  let Inst{11-8} = 0b1111;
2848  let Inst{7} = fc{0};
2849  let Inst{6} = 0b0;
2850  let Inst{5} = Qm{3};
2851  let Inst{4} = 0b0;
2852  let Inst{3-1} = Qm{2-0};
2853  let Inst{0} = fc{1};
2854
2855  let Constraints = "";
2856
2857  // We need a custom decoder method for these instructions because of
2858  // the output VCCR operand, which isn't encoded in the instruction
2859  // bits anywhere (there is only one choice for it) but has to be
2860  // included in the MC operands so that codegen will be able to track
2861  // its data flow between instructions, spill/reload it when
2862  // necessary, etc. There seems to be no way to get the Tablegen
2863  // decoder to emit an operand that isn't affected by any instruction
2864  // bit.
2865  let DecoderMethod = "DecodeMVEVCMP<false," # predtype.DecoderMethod # ">";
2866}
2867
2868class MVE_VCMPqqf<string suffix, bit size>
2869    : MVE_VCMPqq<suffix, size, 0b11, pred_basic_fp> {
2870  let Predicates = [HasMVEFloat];
2871}
2872
2873class MVE_VCMPqqi<string suffix, bits<2> size>
2874    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_i> {
2875  let Inst{12} = 0b0;
2876  let Inst{0} = 0b0;
2877}
2878
2879class MVE_VCMPqqu<string suffix, bits<2> size>
2880    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_u> {
2881  let Inst{12} = 0b0;
2882  let Inst{0} = 0b1;
2883}
2884
2885class MVE_VCMPqqs<string suffix, bits<2> size>
2886    : MVE_VCMPqq<suffix, 0b1, size, pred_basic_s> {
2887  let Inst{12} = 0b1;
2888}
2889
2890def MVE_VCMPf32 : MVE_VCMPqqf<"f32", 0b0>;
2891def MVE_VCMPf16 : MVE_VCMPqqf<"f16", 0b1>;
2892
2893def MVE_VCMPi8  : MVE_VCMPqqi<"i8",  0b00>;
2894def MVE_VCMPi16 : MVE_VCMPqqi<"i16", 0b01>;
2895def MVE_VCMPi32 : MVE_VCMPqqi<"i32", 0b10>;
2896
2897def MVE_VCMPu8  : MVE_VCMPqqu<"u8",  0b00>;
2898def MVE_VCMPu16 : MVE_VCMPqqu<"u16", 0b01>;
2899def MVE_VCMPu32 : MVE_VCMPqqu<"u32", 0b10>;
2900
2901def MVE_VCMPs8  : MVE_VCMPqqs<"s8",  0b00>;
2902def MVE_VCMPs16 : MVE_VCMPqqs<"s16", 0b01>;
2903def MVE_VCMPs32 : MVE_VCMPqqs<"s32", 0b10>;
2904
2905class MVE_VCMPqr<string suffix, bit bit_28, bits<2> bits_21_20,
2906                 VCMPPredicateOperand predtype, list<dag> pattern=[]>
2907  : MVE_p<(outs VCCR:$P0), (ins MQPR:$Qn, GPRwithZR:$Rm, predtype:$fc),
2908           NoItinerary, "vcmp", suffix, "$fc, $Qn, $Rm", vpred_n, "", pattern> {
2909  // Base class for comparing a vector register with a scalar
2910  bits<3> fc;
2911  bits<4> Qn;
2912  bits<4> Rm;
2913
2914  let Inst{28} = bit_28;
2915  let Inst{25-22} = 0b1000;
2916  let Inst{21-20} = bits_21_20;
2917  let Inst{19-17} = Qn{2-0};
2918  let Inst{16-13} = 0b1000;
2919  let Inst{12} = fc{2};
2920  let Inst{11-8} = 0b1111;
2921  let Inst{7} = fc{0};
2922  let Inst{6} = 0b1;
2923  let Inst{5} = fc{1};
2924  let Inst{4} = 0b0;
2925  let Inst{3-0} = Rm{3-0};
2926
2927  let Constraints = "";
2928  // Custom decoder method, for the same reason as MVE_VCMPqq
2929  let DecoderMethod = "DecodeMVEVCMP<true," # predtype.DecoderMethod # ">";
2930}
2931
2932class MVE_VCMPqrf<string suffix, bit size>
2933    : MVE_VCMPqr<suffix, size, 0b11, pred_basic_fp> {
2934  let Predicates = [HasMVEFloat];
2935}
2936
2937class MVE_VCMPqri<string suffix, bits<2> size>
2938    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_i> {
2939  let Inst{12} = 0b0;
2940  let Inst{5} = 0b0;
2941}
2942
2943class MVE_VCMPqru<string suffix, bits<2> size>
2944    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_u> {
2945  let Inst{12} = 0b0;
2946  let Inst{5} = 0b1;
2947}
2948
2949class MVE_VCMPqrs<string suffix, bits<2> size>
2950    : MVE_VCMPqr<suffix, 0b1, size, pred_basic_s> {
2951  let Inst{12} = 0b1;
2952}
2953
2954def MVE_VCMPf32r : MVE_VCMPqrf<"f32", 0b0>;
2955def MVE_VCMPf16r : MVE_VCMPqrf<"f16", 0b1>;
2956
2957def MVE_VCMPi8r  : MVE_VCMPqri<"i8",  0b00>;
2958def MVE_VCMPi16r : MVE_VCMPqri<"i16", 0b01>;
2959def MVE_VCMPi32r : MVE_VCMPqri<"i32", 0b10>;
2960
2961def MVE_VCMPu8r  : MVE_VCMPqru<"u8",  0b00>;
2962def MVE_VCMPu16r : MVE_VCMPqru<"u16", 0b01>;
2963def MVE_VCMPu32r : MVE_VCMPqru<"u32", 0b10>;
2964
2965def MVE_VCMPs8r  : MVE_VCMPqrs<"s8",  0b00>;
2966def MVE_VCMPs16r : MVE_VCMPqrs<"s16", 0b01>;
2967def MVE_VCMPs32r : MVE_VCMPqrs<"s32", 0b10>;
2968
2969// end of MVE compares
2970
2971// start of MVE_qDest_qSrc
2972
2973class MVE_qDest_qSrc<string iname, string suffix, dag oops, dag iops,
2974                     string ops, vpred_ops vpred, string cstr,
2975                     list<dag> pattern=[]>
2976  : MVE_p<oops, iops, NoItinerary, iname, suffix,
2977          ops, vpred, cstr, pattern> {
2978  bits<4> Qd;
2979  bits<4> Qm;
2980
2981  let Inst{25-23} = 0b100;
2982  let Inst{22} = Qd{3};
2983  let Inst{15-13} = Qd{2-0};
2984  let Inst{11-9} = 0b111;
2985  let Inst{6} = 0b0;
2986  let Inst{5} = Qm{3};
2987  let Inst{4} = 0b0;
2988  let Inst{3-1} = Qm{2-0};
2989}
2990
2991class MVE_VQxDMLxDH<string iname, bit exch, bit round, bit subtract,
2992                    string suffix, bits<2> size, list<dag> pattern=[]>
2993  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
2994                   (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
2995                   vpred_n, "$Qd = $Qd_src", pattern> {
2996  bits<4> Qn;
2997
2998  let Inst{28} = subtract;
2999  let Inst{21-20} = size;
3000  let Inst{19-17} = Qn{2-0};
3001  let Inst{16} = 0b0;
3002  let Inst{12} = exch;
3003  let Inst{8} = 0b0;
3004  let Inst{7} = Qn{3};
3005  let Inst{0} = round;
3006}
3007
3008multiclass MVE_VQxDMLxDH_multi<string iname, bit exch,
3009                               bit round, bit subtract> {
3010  def s8  : MVE_VQxDMLxDH<iname, exch, round, subtract, "s8",  0b00>;
3011  def s16 : MVE_VQxDMLxDH<iname, exch, round, subtract, "s16", 0b01>;
3012  def s32 : MVE_VQxDMLxDH<iname, exch, round, subtract, "s32", 0b10>;
3013}
3014
3015defm MVE_VQDMLADH   : MVE_VQxDMLxDH_multi<"vqdmladh",   0b0, 0b0, 0b0>;
3016defm MVE_VQDMLADHX  : MVE_VQxDMLxDH_multi<"vqdmladhx",  0b1, 0b0, 0b0>;
3017defm MVE_VQRDMLADH  : MVE_VQxDMLxDH_multi<"vqrdmladh",  0b0, 0b1, 0b0>;
3018defm MVE_VQRDMLADHX : MVE_VQxDMLxDH_multi<"vqrdmladhx", 0b1, 0b1, 0b0>;
3019defm MVE_VQDMLSDH   : MVE_VQxDMLxDH_multi<"vqdmlsdh",   0b0, 0b0, 0b1>;
3020defm MVE_VQDMLSDHX  : MVE_VQxDMLxDH_multi<"vqdmlsdhx",  0b1, 0b0, 0b1>;
3021defm MVE_VQRDMLSDH  : MVE_VQxDMLxDH_multi<"vqrdmlsdh",  0b0, 0b1, 0b1>;
3022defm MVE_VQRDMLSDHX : MVE_VQxDMLxDH_multi<"vqrdmlsdhx", 0b1, 0b1, 0b1>;
3023
3024class MVE_VCMUL<string iname, string suffix, bit size, list<dag> pattern=[]>
3025  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
3026                   (ins MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
3027                   "$Qd, $Qn, $Qm, $rot", vpred_r, "", pattern> {
3028  bits<4> Qn;
3029  bits<2> rot;
3030
3031  let Inst{28} = size;
3032  let Inst{21-20} = 0b11;
3033  let Inst{19-17} = Qn{2-0};
3034  let Inst{16} = 0b0;
3035  let Inst{12} = rot{1};
3036  let Inst{8} = 0b0;
3037  let Inst{7} = Qn{3};
3038  let Inst{0} = rot{0};
3039
3040  let Predicates = [HasMVEFloat];
3041}
3042
3043def MVE_VCMULf16 : MVE_VCMUL<"vcmul", "f16", 0b0>;
3044def MVE_VCMULf32 : MVE_VCMUL<"vcmul", "f32", 0b1>;
3045
3046class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,
3047                bit T, list<dag> pattern=[]>
3048  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
3049                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
3050                   vpred_r, "", pattern> {
3051  bits<4> Qd;
3052  bits<4> Qn;
3053  bits<4> Qm;
3054
3055  let Inst{28} = bit_28;
3056  let Inst{21-20} = bits_21_20;
3057  let Inst{19-17} = Qn{2-0};
3058  let Inst{16} = 0b1;
3059  let Inst{12} = T;
3060  let Inst{8} = 0b0;
3061  let Inst{7} = Qn{3};
3062  let Inst{0} = 0b0;
3063}
3064
3065multiclass MVE_VMULL_multi<string iname, string suffix,
3066                           bit bit_28, bits<2> bits_21_20> {
3067  def bh : MVE_VMULL<iname # "b", suffix, bit_28, bits_21_20, 0b0>;
3068  def th : MVE_VMULL<iname # "t", suffix, bit_28, bits_21_20, 0b1>;
3069}
3070
3071// For integer multiplies, bits 21:20 encode size, and bit 28 signedness.
3072// For polynomial multiplies, bits 21:20 take the unused value 0b11, and
3073// bit 28 switches to encoding the size.
3074
3075defm MVE_VMULLs8  : MVE_VMULL_multi<"vmull", "s8",  0b0, 0b00>;
3076defm MVE_VMULLs16 : MVE_VMULL_multi<"vmull", "s16", 0b0, 0b01>;
3077defm MVE_VMULLs32 : MVE_VMULL_multi<"vmull", "s32", 0b0, 0b10>;
3078defm MVE_VMULLu8  : MVE_VMULL_multi<"vmull", "u8",  0b1, 0b00>;
3079defm MVE_VMULLu16 : MVE_VMULL_multi<"vmull", "u16", 0b1, 0b01>;
3080defm MVE_VMULLu32 : MVE_VMULL_multi<"vmull", "u32", 0b1, 0b10>;
3081defm MVE_VMULLp8  : MVE_VMULL_multi<"vmull", "p8",  0b0, 0b11>;
3082defm MVE_VMULLp16 : MVE_VMULL_multi<"vmull", "p16", 0b1, 0b11>;
3083
3084class MVE_VxMULH<string iname, string suffix, bit U, bits<2> size,
3085                 bit round, list<dag> pattern=[]>
3086  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
3087                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
3088                   vpred_r, "", pattern> {
3089  bits<4> Qn;
3090
3091  let Inst{28} = U;
3092  let Inst{21-20} = size;
3093  let Inst{19-17} = Qn{2-0};
3094  let Inst{16} = 0b1;
3095  let Inst{12} = round;
3096  let Inst{8} = 0b0;
3097  let Inst{7} = Qn{3};
3098  let Inst{0} = 0b1;
3099}
3100
3101def MVE_VMULHs8   : MVE_VxMULH<"vmulh",  "s8",  0b0, 0b00, 0b0>;
3102def MVE_VMULHs16  : MVE_VxMULH<"vmulh",  "s16", 0b0, 0b01, 0b0>;
3103def MVE_VMULHs32  : MVE_VxMULH<"vmulh",  "s32", 0b0, 0b10, 0b0>;
3104def MVE_VMULHu8   : MVE_VxMULH<"vmulh",  "u8",  0b1, 0b00, 0b0>;
3105def MVE_VMULHu16  : MVE_VxMULH<"vmulh",  "u16", 0b1, 0b01, 0b0>;
3106def MVE_VMULHu32  : MVE_VxMULH<"vmulh",  "u32", 0b1, 0b10, 0b0>;
3107
3108def MVE_VRMULHs8  : MVE_VxMULH<"vrmulh", "s8",  0b0, 0b00, 0b1>;
3109def MVE_VRMULHs16 : MVE_VxMULH<"vrmulh", "s16", 0b0, 0b01, 0b1>;
3110def MVE_VRMULHs32 : MVE_VxMULH<"vrmulh", "s32", 0b0, 0b10, 0b1>;
3111def MVE_VRMULHu8  : MVE_VxMULH<"vrmulh", "u8",  0b1, 0b00, 0b1>;
3112def MVE_VRMULHu16 : MVE_VxMULH<"vrmulh", "u16", 0b1, 0b01, 0b1>;
3113def MVE_VRMULHu32 : MVE_VxMULH<"vrmulh", "u32", 0b1, 0b10, 0b1>;
3114
3115class MVE_VxMOVxN<string iname, string suffix, bit bit_28, bit bit_17,
3116                  bits<2> size, bit T, list<dag> pattern=[]>
3117  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
3118                   (ins MQPR:$Qd_src, MQPR:$Qm), "$Qd, $Qm",
3119                   vpred_n, "$Qd = $Qd_src", pattern> {
3120
3121  let Inst{28} = bit_28;
3122  let Inst{21-20} = 0b11;
3123  let Inst{19-18} = size;
3124  let Inst{17} = bit_17;
3125  let Inst{16} = 0b1;
3126  let Inst{12} = T;
3127  let Inst{8} = 0b0;
3128  let Inst{7} = !if(!eq(bit_17, 0), 1, 0);
3129  let Inst{0} = 0b1;
3130}
3131
3132multiclass MVE_VxMOVxN_halves<string iname, string suffix,
3133                              bit bit_28, bit bit_17, bits<2> size> {
3134  def bh : MVE_VxMOVxN<iname # "b", suffix, bit_28, bit_17, size, 0b0>;
3135  def th : MVE_VxMOVxN<iname # "t", suffix, bit_28, bit_17, size, 0b1>;
3136}
3137
3138defm MVE_VMOVNi16   : MVE_VxMOVxN_halves<"vmovn",   "i16", 0b1, 0b0, 0b00>;
3139defm MVE_VMOVNi32   : MVE_VxMOVxN_halves<"vmovn",   "i32", 0b1, 0b0, 0b01>;
3140defm MVE_VQMOVNs16  : MVE_VxMOVxN_halves<"vqmovn",  "s16", 0b0, 0b1, 0b00>;
3141defm MVE_VQMOVNs32  : MVE_VxMOVxN_halves<"vqmovn",  "s32", 0b0, 0b1, 0b01>;
3142defm MVE_VQMOVNu16  : MVE_VxMOVxN_halves<"vqmovn",  "u16", 0b1, 0b1, 0b00>;
3143defm MVE_VQMOVNu32  : MVE_VxMOVxN_halves<"vqmovn",  "u32", 0b1, 0b1, 0b01>;
3144defm MVE_VQMOVUNs16 : MVE_VxMOVxN_halves<"vqmovun", "s16", 0b0, 0b0, 0b00>;
3145defm MVE_VQMOVUNs32 : MVE_VxMOVxN_halves<"vqmovun", "s32", 0b0, 0b0, 0b01>;
3146
3147class MVE_VCVT_ff<string iname, string suffix, bit op, bit T,
3148                  list<dag> pattern=[]>
3149  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qm),
3150                   "$Qd, $Qm", vpred_n, "$Qd = $Qd_src", pattern> {
3151  let Inst{28} = op;
3152  let Inst{21-16} = 0b111111;
3153  let Inst{12} = T;
3154  let Inst{8-7} = 0b00;
3155  let Inst{0} = 0b1;
3156
3157  let Predicates = [HasMVEFloat];
3158}
3159
3160multiclass MVE_VCVT_ff_halves<string suffix, bit op> {
3161  def bh : MVE_VCVT_ff<"vcvtb", suffix, op, 0b0>;
3162  def th : MVE_VCVT_ff<"vcvtt", suffix, op, 0b1>;
3163}
3164
3165defm MVE_VCVTf16f32 : MVE_VCVT_ff_halves<"f16.f32", 0b0>;
3166defm MVE_VCVTf32f16 : MVE_VCVT_ff_halves<"f32.f16", 0b1>;
3167
3168class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
3169                 list<dag> pattern=[]>
3170  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
3171                   (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
3172                   "$Qd, $Qn, $Qm, $rot", vpred_r, "",
3173          pattern> {
3174  bits<4> Qn;
3175  bit rot;
3176
3177  let Inst{28} = halve;
3178  let Inst{21-20} = size;
3179  let Inst{19-17} = Qn{2-0};
3180  let Inst{16} = 0b0;
3181  let Inst{12} = rot;
3182  let Inst{8} = 0b1;
3183  let Inst{7} = Qn{3};
3184  let Inst{0} = 0b0;
3185}
3186
3187def MVE_VCADDi8   : MVE_VxCADD<"vcadd", "i8", 0b00, 0b1>;
3188def MVE_VCADDi16  : MVE_VxCADD<"vcadd", "i16", 0b01, 0b1>;
3189def MVE_VCADDi32  : MVE_VxCADD<"vcadd", "i32", 0b10, 0b1>;
3190
3191def MVE_VHCADDs8  : MVE_VxCADD<"vhcadd", "s8", 0b00, 0b0>;
3192def MVE_VHCADDs16 : MVE_VxCADD<"vhcadd", "s16", 0b01, 0b0>;
3193def MVE_VHCADDs32 : MVE_VxCADD<"vhcadd", "s32", 0b10, 0b0>;
3194
3195class MVE_VADCSBC<string iname, bit I, bit subtract,
3196                  dag carryin, list<dag> pattern=[]>
3197  : MVE_qDest_qSrc<iname, "i32", (outs MQPR:$Qd, cl_FPSCR_NZCV:$carryout),
3198                   !con((ins MQPR:$Qn, MQPR:$Qm), carryin),
3199                   "$Qd, $Qn, $Qm", vpred_r, "", pattern> {
3200  bits<4> Qn;
3201
3202  let Inst{28} = subtract;
3203  let Inst{21-20} = 0b11;
3204  let Inst{19-17} = Qn{2-0};
3205  let Inst{16} = 0b0;
3206  let Inst{12} = I;
3207  let Inst{8} = 0b1;
3208  let Inst{7} = Qn{3};
3209  let Inst{0} = 0b0;
3210
3211  // Custom decoder method in order to add the FPSCR operand(s), which
3212  // Tablegen won't do right
3213  let DecoderMethod = "DecodeMVEVADCInstruction";
3214}
3215
3216def MVE_VADC  : MVE_VADCSBC<"vadc",  0b0, 0b0, (ins cl_FPSCR_NZCV:$carryin)>;
3217def MVE_VADCI : MVE_VADCSBC<"vadci", 0b1, 0b0, (ins)>;
3218
3219def MVE_VSBC  : MVE_VADCSBC<"vsbc",  0b0, 0b1, (ins cl_FPSCR_NZCV:$carryin)>;
3220def MVE_VSBCI : MVE_VADCSBC<"vsbci", 0b1, 0b1, (ins)>;
3221
3222class MVE_VQDMULL<string iname, string suffix, bit size, bit T,
3223                  list<dag> pattern=[]>
3224  : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
3225                   (ins MQPR:$Qn, MQPR:$Qm), "$Qd, $Qn, $Qm",
3226                   vpred_r, "", pattern> {
3227  bits<4> Qn;
3228
3229  let Inst{28} = size;
3230  let Inst{21-20} = 0b11;
3231  let Inst{19-17} = Qn{2-0};
3232  let Inst{16} = 0b0;
3233  let Inst{12} = T;
3234  let Inst{8} = 0b1;
3235  let Inst{7} = Qn{3};
3236  let Inst{0} = 0b1;
3237}
3238
3239multiclass MVE_VQDMULL_halves<string suffix, bit size> {
3240  def bh : MVE_VQDMULL<"vqdmullb", suffix, size, 0b0>;
3241  def th : MVE_VQDMULL<"vqdmullt", suffix, size, 0b1>;
3242}
3243
3244defm MVE_VQDMULLs16 : MVE_VQDMULL_halves<"s16", 0b0>;
3245defm MVE_VQDMULLs32 : MVE_VQDMULL_halves<"s32", 0b1>;
3246
3247// end of mve_qDest_qSrc
3248
3249// start of mve_qDest_rSrc
3250
3251class MVE_qr_base<dag oops, dag iops, InstrItinClass itin, string iname,
3252                  string suffix, string ops, vpred_ops vpred, string cstr,
3253                  list<dag> pattern=[]>
3254   : MVE_p<oops, iops, NoItinerary, iname, suffix, ops, vpred, cstr, pattern> {
3255  bits<4> Qd;
3256  bits<4> Qn;
3257  bits<4> Rm;
3258
3259  let Inst{25-23} = 0b100;
3260  let Inst{22} = Qd{3};
3261  let Inst{19-17} = Qn{2-0};
3262  let Inst{15-13} = Qd{2-0};
3263  let Inst{11-9} = 0b111;
3264  let Inst{7} = Qn{3};
3265  let Inst{6} = 0b1;
3266  let Inst{4} = 0b0;
3267  let Inst{3-0} = Rm{3-0};
3268}
3269
3270class MVE_qDest_rSrc<string iname, string suffix, list<dag> pattern=[]>
3271  : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qn, rGPR:$Rm),
3272          NoItinerary, iname, suffix, "$Qd, $Qn, $Rm", vpred_r, "",
3273           pattern>;
3274
3275class MVE_qDestSrc_rSrc<string iname, string suffix, list<dag> pattern=[]>
3276  : MVE_qr_base<(outs MQPR:$Qd), (ins MQPR:$Qd_src, MQPR:$Qn, rGPR:$Rm),
3277          NoItinerary, iname, suffix, "$Qd, $Qn, $Rm", vpred_n, "$Qd = $Qd_src",
3278           pattern>;
3279
3280class MVE_qDest_single_rSrc<string iname, string suffix, list<dag> pattern=[]>
3281  : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qd_src, rGPR:$Rm), NoItinerary, iname,
3282          suffix, "$Qd, $Rm", vpred_n, "$Qd = $Qd_src", pattern> {
3283  bits<4> Qd;
3284  bits<4> Rm;
3285
3286  let Inst{22} = Qd{3};
3287  let Inst{15-13} = Qd{2-0};
3288  let Inst{3-0} = Rm{3-0};
3289}
3290
3291class MVE_VADDSUB_qr<string iname, string suffix, bits<2> size,
3292                     bit bit_5, bit bit_12, bit bit_16,
3293                     bit bit_28, list<dag> pattern=[]>
3294  : MVE_qDest_rSrc<iname, suffix, pattern> {
3295
3296  let Inst{28} = bit_28;
3297  let Inst{21-20} = size;
3298  let Inst{16} = bit_16;
3299  let Inst{12} = bit_12;
3300  let Inst{8} = 0b1;
3301  let Inst{5} = bit_5;
3302}
3303
3304multiclass MVE_VADDSUB_qr_sizes<string iname, string suffix,
3305                                bit bit_5, bit bit_12, bit bit_16,
3306                                bit bit_28, list<dag> pattern=[]> {
3307  def "8"  : MVE_VADDSUB_qr<iname, suffix#"8",  0b00,
3308                            bit_5, bit_12, bit_16, bit_28>;
3309  def "16" : MVE_VADDSUB_qr<iname, suffix#"16", 0b01,
3310                            bit_5, bit_12, bit_16, bit_28>;
3311  def "32" : MVE_VADDSUB_qr<iname, suffix#"32", 0b10,
3312                            bit_5, bit_12, bit_16, bit_28>;
3313}
3314
3315defm MVE_VADD_qr_i  : MVE_VADDSUB_qr_sizes<"vadd",  "i", 0b0, 0b0, 0b1, 0b0>;
3316defm MVE_VQADD_qr_s : MVE_VADDSUB_qr_sizes<"vqadd", "s", 0b1, 0b0, 0b0, 0b0>;
3317defm MVE_VQADD_qr_u : MVE_VADDSUB_qr_sizes<"vqadd", "u", 0b1, 0b0, 0b0, 0b1>;
3318
3319defm MVE_VSUB_qr_i  : MVE_VADDSUB_qr_sizes<"vsub",  "i", 0b0, 0b1, 0b1, 0b0>;
3320defm MVE_VQSUB_qr_s : MVE_VADDSUB_qr_sizes<"vqsub", "s", 0b1, 0b1, 0b0, 0b0>;
3321defm MVE_VQSUB_qr_u : MVE_VADDSUB_qr_sizes<"vqsub", "u", 0b1, 0b1, 0b0, 0b1>;
3322
3323class MVE_VQDMULL_qr<string iname, string suffix, bit size,
3324                     bit T, list<dag> pattern=[]>
3325  : MVE_qDest_rSrc<iname, suffix, pattern> {
3326
3327  let Inst{28} = size;
3328  let Inst{21-20} = 0b11;
3329  let Inst{16} = 0b0;
3330  let Inst{12} = T;
3331  let Inst{8} = 0b1;
3332  let Inst{5} = 0b1;
3333}
3334
3335multiclass MVE_VQDMULL_qr_halves<string suffix, bit size> {
3336  def bh : MVE_VQDMULL_qr<"vqdmullb", suffix, size, 0b0>;
3337  def th : MVE_VQDMULL_qr<"vqdmullt", suffix, size, 0b1>;
3338}
3339
3340defm MVE_VQDMULL_qr_s16 : MVE_VQDMULL_qr_halves<"s16", 0b0>;
3341defm MVE_VQDMULL_qr_s32 : MVE_VQDMULL_qr_halves<"s32", 0b1>;
3342
3343class MVE_VxADDSUB_qr<string iname, string suffix,
3344                      bit bit_28, bits<2> bits_21_20, bit subtract,
3345                      list<dag> pattern=[]>
3346  : MVE_qDest_rSrc<iname, suffix, pattern> {
3347
3348  let Inst{28} = bit_28;
3349  let Inst{21-20} = bits_21_20;
3350  let Inst{16} = 0b0;
3351  let Inst{12} = subtract;
3352  let Inst{8} = 0b1;
3353  let Inst{5} = 0b0;
3354}
3355
3356def MVE_VHADD_qr_s8   : MVE_VxADDSUB_qr<"vhadd", "s8",  0b0, 0b00, 0b0>;
3357def MVE_VHADD_qr_s16  : MVE_VxADDSUB_qr<"vhadd", "s16", 0b0, 0b01, 0b0>;
3358def MVE_VHADD_qr_s32  : MVE_VxADDSUB_qr<"vhadd", "s32", 0b0, 0b10, 0b0>;
3359def MVE_VHADD_qr_u8   : MVE_VxADDSUB_qr<"vhadd", "u8",  0b1, 0b00, 0b0>;
3360def MVE_VHADD_qr_u16  : MVE_VxADDSUB_qr<"vhadd", "u16", 0b1, 0b01, 0b0>;
3361def MVE_VHADD_qr_u32  : MVE_VxADDSUB_qr<"vhadd", "u32", 0b1, 0b10, 0b0>;
3362
3363def MVE_VHSUB_qr_s8   : MVE_VxADDSUB_qr<"vhsub", "s8",  0b0, 0b00, 0b1>;
3364def MVE_VHSUB_qr_s16  : MVE_VxADDSUB_qr<"vhsub", "s16", 0b0, 0b01, 0b1>;
3365def MVE_VHSUB_qr_s32  : MVE_VxADDSUB_qr<"vhsub", "s32", 0b0, 0b10, 0b1>;
3366def MVE_VHSUB_qr_u8   : MVE_VxADDSUB_qr<"vhsub", "u8",  0b1, 0b00, 0b1>;
3367def MVE_VHSUB_qr_u16  : MVE_VxADDSUB_qr<"vhsub", "u16", 0b1, 0b01, 0b1>;
3368def MVE_VHSUB_qr_u32  : MVE_VxADDSUB_qr<"vhsub", "u32", 0b1, 0b10, 0b1>;
3369
3370let Predicates = [HasMVEFloat] in {
3371  def MVE_VADD_qr_f32 : MVE_VxADDSUB_qr<"vadd",  "f32", 0b0, 0b11, 0b0>;
3372  def MVE_VADD_qr_f16 : MVE_VxADDSUB_qr<"vadd",  "f16", 0b1, 0b11, 0b0>;
3373
3374  def MVE_VSUB_qr_f32 : MVE_VxADDSUB_qr<"vsub",  "f32", 0b0, 0b11, 0b1>;
3375  def MVE_VSUB_qr_f16 : MVE_VxADDSUB_qr<"vsub",  "f16", 0b1, 0b11, 0b1>;
3376}
3377
3378class MVE_VxSHL_qr<string iname, string suffix, bit U, bits<2> size,
3379                   bit bit_7, bit bit_17, list<dag> pattern=[]>
3380  : MVE_qDest_single_rSrc<iname, suffix, pattern> {
3381
3382  let Inst{28} = U;
3383  let Inst{25-23} = 0b100;
3384  let Inst{21-20} = 0b11;
3385  let Inst{19-18} = size;
3386  let Inst{17} = bit_17;
3387  let Inst{16} = 0b1;
3388  let Inst{12-8} = 0b11110;
3389  let Inst{7} = bit_7;
3390  let Inst{6-4} = 0b110;
3391}
3392
3393multiclass MVE_VxSHL_qr_types<string iname, bit bit_7, bit bit_17> {
3394  def s8  : MVE_VxSHL_qr<iname, "s8", 0b0, 0b00, bit_7, bit_17>;
3395  def s16 : MVE_VxSHL_qr<iname, "s16", 0b0, 0b01, bit_7, bit_17>;
3396  def s32 : MVE_VxSHL_qr<iname, "s32", 0b0, 0b10, bit_7, bit_17>;
3397  def u8  : MVE_VxSHL_qr<iname, "u8", 0b1, 0b00, bit_7, bit_17>;
3398  def u16 : MVE_VxSHL_qr<iname, "u16", 0b1, 0b01, bit_7, bit_17>;
3399  def u32 : MVE_VxSHL_qr<iname, "u32", 0b1, 0b10, bit_7, bit_17>;
3400}
3401
3402defm MVE_VSHL_qr   : MVE_VxSHL_qr_types<"vshl",   0b0, 0b0>;
3403defm MVE_VRSHL_qr  : MVE_VxSHL_qr_types<"vrshl",  0b0, 0b1>;
3404defm MVE_VQSHL_qr  : MVE_VxSHL_qr_types<"vqshl",  0b1, 0b0>;
3405defm MVE_VQRSHL_qr : MVE_VxSHL_qr_types<"vqrshl", 0b1, 0b1>;
3406
3407let Predicates = [HasMVEInt] in {
3408  def : Pat<(v4i32 (ARMvshlu (v4i32 MQPR:$Qm), (v4i32 (ARMvdup GPR:$Rm)))),
3409            (v4i32 (MVE_VSHL_qru32 (v4i32 MQPR:$Qm), GPR:$Rm))>;
3410  def : Pat<(v8i16 (ARMvshlu (v8i16 MQPR:$Qm), (v8i16 (ARMvdup GPR:$Rm)))),
3411            (v8i16 (MVE_VSHL_qru16 (v8i16 MQPR:$Qm), GPR:$Rm))>;
3412  def : Pat<(v16i8 (ARMvshlu (v16i8 MQPR:$Qm), (v16i8 (ARMvdup GPR:$Rm)))),
3413            (v16i8 (MVE_VSHL_qru8 (v16i8 MQPR:$Qm), GPR:$Rm))>;
3414
3415  def : Pat<(v4i32 (ARMvshls (v4i32 MQPR:$Qm), (v4i32 (ARMvdup GPR:$Rm)))),
3416            (v4i32 (MVE_VSHL_qrs32 (v4i32 MQPR:$Qm), GPR:$Rm))>;
3417  def : Pat<(v8i16 (ARMvshls (v8i16 MQPR:$Qm), (v8i16 (ARMvdup GPR:$Rm)))),
3418            (v8i16 (MVE_VSHL_qrs16 (v8i16 MQPR:$Qm), GPR:$Rm))>;
3419  def : Pat<(v16i8 (ARMvshls (v16i8 MQPR:$Qm), (v16i8 (ARMvdup GPR:$Rm)))),
3420            (v16i8 (MVE_VSHL_qrs8 (v16i8 MQPR:$Qm), GPR:$Rm))>;
3421}
3422
3423class MVE_VBRSR<string iname, string suffix, bits<2> size, list<dag> pattern=[]>
3424  : MVE_qDest_rSrc<iname, suffix, pattern> {
3425
3426  let Inst{28} = 0b1;
3427  let Inst{21-20} = size;
3428  let Inst{16} = 0b1;
3429  let Inst{12} = 0b1;
3430  let Inst{8} = 0b0;
3431  let Inst{5} = 0b1;
3432}
3433
3434def MVE_VBRSR8  : MVE_VBRSR<"vbrsr", "8", 0b00>;
3435def MVE_VBRSR16 : MVE_VBRSR<"vbrsr", "16", 0b01>;
3436def MVE_VBRSR32 : MVE_VBRSR<"vbrsr", "32", 0b10>;
3437
3438class MVE_VMUL_qr_int<string iname, string suffix,
3439                      bits<2> size, list<dag> pattern=[]>
3440  : MVE_qDest_rSrc<iname, suffix, pattern> {
3441
3442  let Inst{28} = 0b0;
3443  let Inst{21-20} = size;
3444  let Inst{16} = 0b1;
3445  let Inst{12} = 0b1;
3446  let Inst{8} = 0b0;
3447  let Inst{5} = 0b1;
3448}
3449
3450def MVE_VMUL_qr_i8  : MVE_VMUL_qr_int<"vmul", "i8",  0b00>;
3451def MVE_VMUL_qr_i16 : MVE_VMUL_qr_int<"vmul", "i16", 0b01>;
3452def MVE_VMUL_qr_i32 : MVE_VMUL_qr_int<"vmul", "i32", 0b10>;
3453
3454class MVE_VxxMUL_qr<string iname, string suffix,
3455                    bit bit_28, bits<2> bits_21_20, list<dag> pattern=[]>
3456  : MVE_qDest_rSrc<iname, suffix, pattern> {
3457
3458  let Inst{28} = bit_28;
3459  let Inst{21-20} = bits_21_20;
3460  let Inst{16} = 0b1;
3461  let Inst{12} = 0b0;
3462  let Inst{8} = 0b0;
3463  let Inst{5} = 0b1;
3464}
3465
3466def MVE_VQDMULH_qr_s8   : MVE_VxxMUL_qr<"vqdmulh",  "s8",  0b0, 0b00>;
3467def MVE_VQDMULH_qr_s16  : MVE_VxxMUL_qr<"vqdmulh",  "s16", 0b0, 0b01>;
3468def MVE_VQDMULH_qr_s32  : MVE_VxxMUL_qr<"vqdmulh",  "s32", 0b0, 0b10>;
3469
3470def MVE_VQRDMULH_qr_s8  : MVE_VxxMUL_qr<"vqrdmulh", "s8",  0b1, 0b00>;
3471def MVE_VQRDMULH_qr_s16 : MVE_VxxMUL_qr<"vqrdmulh", "s16", 0b1, 0b01>;
3472def MVE_VQRDMULH_qr_s32 : MVE_VxxMUL_qr<"vqrdmulh", "s32", 0b1, 0b10>;
3473
3474let Predicates = [HasMVEFloat] in {
3475  def MVE_VMUL_qr_f16   : MVE_VxxMUL_qr<"vmul", "f16", 0b1, 0b11>;
3476  def MVE_VMUL_qr_f32   : MVE_VxxMUL_qr<"vmul", "f32", 0b0, 0b11>;
3477}
3478
3479class MVE_VFMAMLA_qr<string iname, string suffix,
3480                   bit bit_28, bits<2> bits_21_20, bit S,
3481                   list<dag> pattern=[]>
3482  : MVE_qDestSrc_rSrc<iname, suffix, pattern> {
3483
3484  let Inst{28} = bit_28;
3485  let Inst{21-20} = bits_21_20;
3486  let Inst{16} = 0b1;
3487  let Inst{12} = S;
3488  let Inst{8} = 0b0;
3489  let Inst{5} = 0b0;
3490}
3491
3492def MVE_VMLA_qr_s8     : MVE_VFMAMLA_qr<"vmla",  "s8",  0b0, 0b00, 0b0>;
3493def MVE_VMLA_qr_s16    : MVE_VFMAMLA_qr<"vmla",  "s16", 0b0, 0b01, 0b0>;
3494def MVE_VMLA_qr_s32    : MVE_VFMAMLA_qr<"vmla",  "s32", 0b0, 0b10, 0b0>;
3495def MVE_VMLA_qr_u8     : MVE_VFMAMLA_qr<"vmla",  "u8",  0b1, 0b00, 0b0>;
3496def MVE_VMLA_qr_u16    : MVE_VFMAMLA_qr<"vmla",  "u16", 0b1, 0b01, 0b0>;
3497def MVE_VMLA_qr_u32    : MVE_VFMAMLA_qr<"vmla",  "u32", 0b1, 0b10, 0b0>;
3498
3499def MVE_VMLAS_qr_s8    : MVE_VFMAMLA_qr<"vmlas", "s8",  0b0, 0b00, 0b1>;
3500def MVE_VMLAS_qr_s16   : MVE_VFMAMLA_qr<"vmlas", "s16", 0b0, 0b01, 0b1>;
3501def MVE_VMLAS_qr_s32   : MVE_VFMAMLA_qr<"vmlas", "s32", 0b0, 0b10, 0b1>;
3502def MVE_VMLAS_qr_u8    : MVE_VFMAMLA_qr<"vmlas", "u8",  0b1, 0b00, 0b1>;
3503def MVE_VMLAS_qr_u16   : MVE_VFMAMLA_qr<"vmlas", "u16", 0b1, 0b01, 0b1>;
3504def MVE_VMLAS_qr_u32   : MVE_VFMAMLA_qr<"vmlas", "u32", 0b1, 0b10, 0b1>;
3505
3506let Predicates = [HasMVEFloat] in {
3507  def MVE_VFMA_qr_f16  : MVE_VFMAMLA_qr<"vfma",  "f16", 0b1, 0b11, 0b0>;
3508  def MVE_VFMA_qr_f32  : MVE_VFMAMLA_qr<"vfma",  "f32", 0b0, 0b11, 0b0>;
3509  def MVE_VFMA_qr_Sf16 : MVE_VFMAMLA_qr<"vfmas", "f16", 0b1, 0b11, 0b1>;
3510  def MVE_VFMA_qr_Sf32 : MVE_VFMAMLA_qr<"vfmas", "f32", 0b0, 0b11, 0b1>;
3511}
3512
3513class MVE_VQDMLAH_qr<string iname, string suffix, bit U, bits<2> size,
3514                     bit bit_5, bit bit_12, list<dag> pattern=[]>
3515  : MVE_qDestSrc_rSrc<iname, suffix, pattern> {
3516
3517  let Inst{28} = U;
3518  let Inst{21-20} = size;
3519  let Inst{16} = 0b0;
3520  let Inst{12} = bit_12;
3521  let Inst{8} = 0b0;
3522  let Inst{5} = bit_5;
3523}
3524
3525multiclass MVE_VQDMLAH_qr_types<string iname, bit bit_5, bit bit_12> {
3526  def s8  : MVE_VQDMLAH_qr<iname, "s8",  0b0, 0b00, bit_5, bit_12>;
3527  def s16 : MVE_VQDMLAH_qr<iname, "s16", 0b0, 0b01, bit_5, bit_12>;
3528  def s32 : MVE_VQDMLAH_qr<iname, "s32", 0b0, 0b10, bit_5, bit_12>;
3529}
3530
3531defm MVE_VQDMLAH_qr   : MVE_VQDMLAH_qr_types<"vqdmlah",   0b1, 0b0>;
3532defm MVE_VQRDMLAH_qr  : MVE_VQDMLAH_qr_types<"vqrdmlah",  0b0, 0b0>;
3533defm MVE_VQDMLASH_qr  : MVE_VQDMLAH_qr_types<"vqdmlash",  0b1, 0b1>;
3534defm MVE_VQRDMLASH_qr : MVE_VQDMLAH_qr_types<"vqrdmlash", 0b0, 0b1>;
3535
3536class MVE_VxDUP<string iname, string suffix, bits<2> size, bit bit_12,
3537              list<dag> pattern=[]>
3538  : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
3539          (ins tGPREven:$Rn_src, MVE_VIDUP_imm:$imm), NoItinerary,
3540          iname, suffix, "$Qd, $Rn, $imm", vpred_r, "$Rn = $Rn_src",
3541          pattern> {
3542  bits<4> Qd;
3543  bits<4> Rn;
3544  bits<2> imm;
3545
3546  let Inst{28} = 0b0;
3547  let Inst{25-23} = 0b100;
3548  let Inst{22} = Qd{3};
3549  let Inst{21-20} = size;
3550  let Inst{19-17} = Rn{3-1};
3551  let Inst{16} = 0b1;
3552  let Inst{15-13} = Qd{2-0};
3553  let Inst{12} = bit_12;
3554  let Inst{11-8} = 0b1111;
3555  let Inst{7} = imm{1};
3556  let Inst{6-1} = 0b110111;
3557  let Inst{0} = imm{0};
3558}
3559
3560def MVE_VIDUPu8  : MVE_VxDUP<"vidup", "u8",  0b00, 0b0>;
3561def MVE_VIDUPu16 : MVE_VxDUP<"vidup", "u16", 0b01, 0b0>;
3562def MVE_VIDUPu32 : MVE_VxDUP<"vidup", "u32", 0b10, 0b0>;
3563
3564def MVE_VDDUPu8  : MVE_VxDUP<"vddup", "u8",  0b00, 0b1>;
3565def MVE_VDDUPu16 : MVE_VxDUP<"vddup", "u16", 0b01, 0b1>;
3566def MVE_VDDUPu32 : MVE_VxDUP<"vddup", "u32", 0b10, 0b1>;
3567
3568class MVE_VxWDUP<string iname, string suffix, bits<2> size, bit bit_12,
3569                 list<dag> pattern=[]>
3570  : MVE_p<(outs MQPR:$Qd, tGPREven:$Rn),
3571          (ins tGPREven:$Rn_src, tGPROdd:$Rm, MVE_VIDUP_imm:$imm), NoItinerary,
3572          iname, suffix, "$Qd, $Rn, $Rm, $imm", vpred_r, "$Rn = $Rn_src",
3573          pattern> {
3574  bits<4> Qd;
3575  bits<4> Rm;
3576  bits<4> Rn;
3577  bits<2> imm;
3578
3579  let Inst{28} = 0b0;
3580  let Inst{25-23} = 0b100;
3581  let Inst{22} = Qd{3};
3582  let Inst{21-20} = size;
3583  let Inst{19-17} = Rn{3-1};
3584  let Inst{16} = 0b1;
3585  let Inst{15-13} = Qd{2-0};
3586  let Inst{12} = bit_12;
3587  let Inst{11-8} = 0b1111;
3588  let Inst{7} = imm{1};
3589  let Inst{6-4} = 0b110;
3590  let Inst{3-1} = Rm{3-1};
3591  let Inst{0} = imm{0};
3592}
3593
3594def MVE_VIWDUPu8  : MVE_VxWDUP<"viwdup", "u8",  0b00, 0b0>;
3595def MVE_VIWDUPu16 : MVE_VxWDUP<"viwdup", "u16", 0b01, 0b0>;
3596def MVE_VIWDUPu32 : MVE_VxWDUP<"viwdup", "u32", 0b10, 0b0>;
3597
3598def MVE_VDWDUPu8  : MVE_VxWDUP<"vdwdup", "u8",  0b00, 0b1>;
3599def MVE_VDWDUPu16 : MVE_VxWDUP<"vdwdup", "u16", 0b01, 0b1>;
3600def MVE_VDWDUPu32 : MVE_VxWDUP<"vdwdup", "u32", 0b10, 0b1>;
3601
3602class MVE_VCTP<string suffix, bits<2> size, list<dag> pattern=[]>
3603  : MVE_p<(outs VCCR:$P0), (ins rGPR:$Rn), NoItinerary, "vctp", suffix,
3604          "$Rn", vpred_n, "", pattern> {
3605  bits<4> Rn;
3606
3607  let Inst{28-27} = 0b10;
3608  let Inst{26-22} = 0b00000;
3609  let Inst{21-20} = size;
3610  let Inst{19-16} = Rn{3-0};
3611  let Inst{15-11} = 0b11101;
3612  let Inst{10-0}  = 0b00000000001;
3613  let Unpredictable{10-0} = 0b11111111111;
3614
3615  let Constraints = "";
3616  let DecoderMethod = "DecodeMveVCTP";
3617}
3618
3619def MVE_VCTP8  : MVE_VCTP<"8",  0b00>;
3620def MVE_VCTP16 : MVE_VCTP<"16", 0b01>;
3621def MVE_VCTP32 : MVE_VCTP<"32", 0b10>;
3622def MVE_VCTP64 : MVE_VCTP<"64", 0b11>;
3623
3624// end of mve_qDest_rSrc
3625
3626// start of coproc mov
3627
3628class MVE_VMOV_64bit<dag oops, dag iops, bit to_qreg, string ops, string cstr>
3629  : MVE_VMOV_lane_base<oops, !con(iops, (ins MVEPairVectorIndex2:$idx,
3630                                             MVEPairVectorIndex0:$idx2)),
3631                       NoItinerary, "vmov", "", ops, cstr, []> {
3632  bits<5> Rt;
3633  bits<5> Rt2;
3634  bits<4> Qd;
3635  bit idx;
3636  bit idx2;
3637
3638  let Inst{31-23} = 0b111011000;
3639  let Inst{22} = Qd{3};
3640  let Inst{21} = 0b0;
3641  let Inst{20} = to_qreg;
3642  let Inst{19-16} = Rt2{3-0};
3643  let Inst{15-13} = Qd{2-0};
3644  let Inst{12-5} = 0b01111000;
3645  let Inst{4} = idx2;
3646  let Inst{3-0} = Rt{3-0};
3647}
3648
3649// The assembly syntax for these instructions mentions the vector
3650// register name twice, e.g.
3651//
3652//    vmov q2[2], q2[0], r0, r1
3653//    vmov r0, r1, q2[2], q2[0]
3654//
3655// which needs a bit of juggling with MC operand handling.
3656//
3657// For the move _into_ a vector register, the MC operand list also has
3658// to mention the register name twice: once as the output, and once as
3659// an extra input to represent where the unchanged half of the output
3660// register comes from (when this instruction is used in code
3661// generation). So we arrange that the first mention of the vector reg
3662// in the instruction is considered by the AsmMatcher to be the output
3663// ($Qd), and the second one is the input ($QdSrc). Binding them
3664// together with the existing 'tie' constraint is enough to enforce at
3665// register allocation time that they have to be the same register.
3666//
3667// For the move _from_ a vector register, there's no way to get round
3668// the fact that both instances of that register name have to be
3669// inputs. They have to be the same register again, but this time, we
3670// can't use a tie constraint, because that has to be between an
3671// output and an input operand. So this time, we have to arrange that
3672// the q-reg appears just once in the MC operand list, in spite of
3673// being mentioned twice in the asm syntax - which needs a custom
3674// AsmMatchConverter.
3675
3676def MVE_VMOV_q_rr : MVE_VMOV_64bit<(outs MQPR:$Qd),
3677                                   (ins MQPR:$QdSrc, rGPR:$Rt, rGPR:$Rt2),
3678                                   0b1, "$Qd$idx, $QdSrc$idx2, $Rt, $Rt2",
3679                                   "$Qd = $QdSrc"> {
3680  let DecoderMethod = "DecodeMVEVMOVDRegtoQ";
3681}
3682
3683def MVE_VMOV_rr_q : MVE_VMOV_64bit<(outs rGPR:$Rt, rGPR:$Rt2), (ins MQPR:$Qd),
3684                                   0b0, "$Rt, $Rt2, $Qd$idx, $Qd$idx2", ""> {
3685  let DecoderMethod = "DecodeMVEVMOVQtoDReg";
3686  let AsmMatchConverter = "cvtMVEVMOVQtoDReg";
3687}
3688
3689// end of coproc mov
3690
3691// start of MVE interleaving load/store
3692
3693// Base class for the family of interleaving/deinterleaving
3694// load/stores with names like VLD20.8 and VST43.32.
3695class MVE_vldst24_base<bit writeback, bit fourregs, bits<2> stage, bits<2> size,
3696                       bit load, dag Oops, dag loadIops, dag wbIops,
3697                       string iname, string ops,
3698                       string cstr, list<dag> pattern=[]>
3699  : MVE_MI<Oops, !con(loadIops, wbIops), NoItinerary, iname, ops, cstr, pattern> {
3700  bits<4> VQd;
3701  bits<4> Rn;
3702
3703  let Inst{31-22} = 0b1111110010;
3704  let Inst{21} = writeback;
3705  let Inst{20} = load;
3706  let Inst{19-16} = Rn;
3707  let Inst{15-13} = VQd{2-0};
3708  let Inst{12-9} = 0b1111;
3709  let Inst{8-7} = size;
3710  let Inst{6-5} = stage;
3711  let Inst{4-1} = 0b0000;
3712  let Inst{0} = fourregs;
3713
3714  let mayLoad = load;
3715  let mayStore = !eq(load,0);
3716}
3717
3718// A parameter class used to encapsulate all the ways the writeback
3719// variants of VLD20 and friends differ from the non-writeback ones.
3720class MVE_vldst24_writeback<bit b, dag Oo, dag Io,
3721                            string sy="", string c="", string n=""> {
3722  bit writeback = b;
3723  dag Oops = Oo;
3724  dag Iops = Io;
3725  string syntax = sy;
3726  string cstr = c;
3727  string id_suffix = n;
3728}
3729
3730// Another parameter class that encapsulates the differences between VLD2x
3731// and VLD4x.
3732class MVE_vldst24_nvecs<int n, list<int> s, bit b, RegisterOperand vl> {
3733  int nvecs = n;
3734  list<int> stages = s;
3735  bit bit0 = b;
3736  RegisterOperand VecList = vl;
3737}
3738
3739// A third parameter class that distinguishes VLDnn.8 from .16 from .32.
3740class MVE_vldst24_lanesize<int i, bits<2> b> {
3741  int lanesize = i;
3742  bits<2> sizebits = b;
3743}
3744
3745// A base class for each direction of transfer: one for load, one for
3746// store. I can't make these a fourth independent parametric tuple
3747// class, because they have to take the nvecs tuple class as a
3748// parameter, in order to find the right VecList operand type.
3749
3750class MVE_vld24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,
3751                     MVE_vldst24_writeback wb, string iname,
3752                     list<dag> pattern=[]>
3753  : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 1,
3754                     !con((outs n.VecList:$VQd), wb.Oops),
3755                     (ins n.VecList:$VQdSrc), wb.Iops,
3756                     iname, "$VQd, $Rn" # wb.syntax,
3757                     wb.cstr # ",$VQdSrc = $VQd", pattern>;
3758
3759class MVE_vst24_base<MVE_vldst24_nvecs n, bits<2> pat, bits<2> size,
3760                     MVE_vldst24_writeback wb, string iname,
3761                     list<dag> pattern=[]>
3762  : MVE_vldst24_base<wb.writeback, n.bit0, pat, size, 0,
3763                     wb.Oops, (ins n.VecList:$VQd), wb.Iops,
3764                     iname, "$VQd, $Rn" # wb.syntax,
3765                     wb.cstr, pattern>;
3766
3767// Actually define all the interleaving loads and stores, by a series
3768// of nested foreaches over number of vectors (VLD2/VLD4); stage
3769// within one of those series (VLDx0/VLDx1/VLDx2/VLDx3); size of
3770// vector lane; writeback or no writeback.
3771foreach n = [MVE_vldst24_nvecs<2, [0,1],     0, VecList2Q>,
3772             MVE_vldst24_nvecs<4, [0,1,2,3], 1, VecList4Q>] in
3773foreach stage = n.stages in
3774foreach s = [MVE_vldst24_lanesize< 8, 0b00>,
3775             MVE_vldst24_lanesize<16, 0b01>,
3776             MVE_vldst24_lanesize<32, 0b10>] in
3777foreach wb = [MVE_vldst24_writeback<
3778                1, (outs rGPR:$wb), (ins t2_nosp_addr_offset_none:$Rn),
3779                "!", "$Rn.base = $wb", "_wb">,
3780              MVE_vldst24_writeback<0, (outs), (ins t2_addr_offset_none:$Rn)>] in {
3781
3782  // For each case within all of those foreaches, define the actual
3783  // instructions. The def names are made by gluing together pieces
3784  // from all the parameter classes, and will end up being things like
3785  // MVE_VLD20_8 and MVE_VST43_16_wb.
3786
3787  def "MVE_VLD" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix
3788    : MVE_vld24_base<n, stage, s.sizebits, wb,
3789                     "vld" # n.nvecs # stage # "." # s.lanesize>;
3790
3791  def "MVE_VST" # n.nvecs # stage # "_" # s.lanesize # wb.id_suffix
3792    : MVE_vst24_base<n, stage, s.sizebits, wb,
3793                     "vst" # n.nvecs # stage # "." # s.lanesize>;
3794}
3795
3796// end of MVE interleaving load/store
3797
3798// start of MVE predicable load/store
3799
3800// A parameter class for the direction of transfer.
3801class MVE_ldst_direction<bit b, dag Oo, dag Io, string c=""> {
3802  bit load = b;
3803  dag Oops = Oo;
3804  dag Iops = Io;
3805  string cstr = c;
3806}
3807def MVE_ld: MVE_ldst_direction<1, (outs MQPR:$Qd), (ins), ",@earlyclobber $Qd">;
3808def MVE_st: MVE_ldst_direction<0, (outs), (ins MQPR:$Qd)>;
3809
3810// A parameter class for the size of memory access in a load.
3811class MVE_memsz<bits<2> e, int s, AddrMode m, string mn, list<string> types> {
3812  bits<2> encoding = e;         // opcode bit(s) for encoding
3813  int shift = s;                // shift applied to immediate load offset
3814  AddrMode AM = m;
3815
3816  // For instruction aliases: define the complete list of type
3817  // suffixes at this size, and the canonical ones for loads and
3818  // stores.
3819  string MnemonicLetter = mn;
3820  int TypeBits = !shl(8, s);
3821  string CanonLoadSuffix = ".u" # TypeBits;
3822  string CanonStoreSuffix = "." # TypeBits;
3823  list<string> suffixes = !foreach(letter, types, "." # letter # TypeBits);
3824}
3825
3826// Instances of MVE_memsz.
3827//
3828// (memD doesn't need an AddrMode, because those are only for
3829// contiguous loads, and memD is only used by gather/scatters.)
3830def MVE_memB: MVE_memsz<0b00, 0, AddrModeT2_i7,   "b", ["", "u", "s"]>;
3831def MVE_memH: MVE_memsz<0b01, 1, AddrModeT2_i7s2, "h", ["", "u", "s", "f"]>;
3832def MVE_memW: MVE_memsz<0b10, 2, AddrModeT2_i7s4, "w", ["", "u", "s", "f"]>;
3833def MVE_memD: MVE_memsz<0b11, 3, ?,               "d", ["", "u", "s", "f"]>;
3834
3835// This is the base class for all the MVE loads and stores other than
3836// the interleaving ones. All the non-interleaving loads/stores share
3837// the characteristic that they operate on just one vector register,
3838// so they are VPT-predicable.
3839//
3840// The predication operand is vpred_n, for both loads and stores. For
3841// store instructions, the reason is obvious: if there is no output
3842// register, there can't be a need for an input parameter giving the
3843// output register's previous value. Load instructions also don't need
3844// that input parameter, because unlike MVE data processing
3845// instructions, predicated loads are defined to set the inactive
3846// lanes of the output register to zero, instead of preserving their
3847// input values.
3848class MVE_VLDRSTR_base<MVE_ldst_direction dir, bit U, bit P, bit W, bit opc,
3849                       dag oops, dag iops, string asm, string suffix,
3850                       string ops, string cstr, list<dag> pattern=[]>
3851 : MVE_p<oops, iops, NoItinerary, asm, suffix, ops, vpred_n, cstr, pattern> {
3852  bits<3> Qd;
3853
3854  let Inst{28} = U;
3855  let Inst{25} = 0b0;
3856  let Inst{24} = P;
3857  let Inst{22} = 0b0;
3858  let Inst{21} = W;
3859  let Inst{20} = dir.load;
3860  let Inst{15-13} = Qd{2-0};
3861  let Inst{12} = opc;
3862  let Inst{11-9} = 0b111;
3863
3864  let mayLoad = dir.load;
3865  let mayStore = !eq(dir.load,0);
3866}
3867
3868// Contiguous load and store instructions. These come in two main
3869// categories: same-size loads/stores in which 128 bits of vector
3870// register is transferred to or from 128 bits of memory in the most
3871// obvious way, and widening loads / narrowing stores, in which the
3872// size of memory accessed is less than the size of a vector register,
3873// so the load instructions sign- or zero-extend each memory value
3874// into a wider vector lane, and the store instructions truncate
3875// correspondingly.
3876//
3877// The instruction mnemonics for these two classes look reasonably
3878// similar, but the actual encodings are different enough to need two
3879// separate base classes.
3880
3881// Contiguous, same size
3882class MVE_VLDRSTR_cs<MVE_ldst_direction dir, MVE_memsz memsz, bit P, bit W,
3883                     dag oops, dag iops, string asm, string suffix,
3884                     IndexMode im, string ops, string cstr>
3885  : MVE_VLDRSTR_base<dir, 0, P, W, 1, oops, iops, asm, suffix, ops, cstr> {
3886  bits<12> addr;
3887  let Inst{23} = addr{7};
3888  let Inst{19-16} = addr{11-8};
3889  let Inst{8-7} = memsz.encoding;
3890  let Inst{6-0} = addr{6-0};
3891}
3892
3893// Contiguous, widening/narrowing
3894class MVE_VLDRSTR_cw<MVE_ldst_direction dir, MVE_memsz memsz, bit U,
3895                     bit P, bit W, bits<2> size, dag oops, dag iops,
3896                     string asm, string suffix, IndexMode im,
3897                     string ops, string cstr>
3898  : MVE_VLDRSTR_base<dir, U, P, W, 0, oops, iops, asm, suffix, ops, cstr> {
3899  bits<11> addr;
3900  let Inst{23} = addr{7};
3901  let Inst{19} = memsz.encoding{0}; // enough to tell 16- from 32-bit
3902  let Inst{18-16} = addr{10-8};
3903  let Inst{8-7} = size;
3904  let Inst{6-0} = addr{6-0};
3905
3906  let IM = im;
3907}
3908
3909// Multiclass wrapper on each of the _cw and _cs base classes, to
3910// generate three writeback modes (none, preindex, postindex).
3911
3912multiclass MVE_VLDRSTR_cw_m<MVE_ldst_direction dir, MVE_memsz memsz,
3913                            string asm, string suffix, bit U, bits<2> size> {
3914  let AM = memsz.AM in {
3915    def "" : MVE_VLDRSTR_cw<
3916        dir, memsz, U, 1, 0, size,
3917        dir.Oops, !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),
3918        asm, suffix, IndexModeNone, "$Qd, $addr", "">;
3919
3920    def _pre : MVE_VLDRSTR_cw<
3921        dir, memsz, U, 1, 1, size,
3922        !con((outs tGPR:$wb), dir.Oops),
3923        !con(dir.Iops, (ins taddrmode_imm7<memsz.shift>:$addr)),
3924        asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {
3925      let DecoderMethod = "DecodeMVE_MEM_1_pre<"#memsz.shift#">";
3926    }
3927
3928    def _post : MVE_VLDRSTR_cw<
3929        dir, memsz, U, 0, 1, size,
3930        !con((outs tGPR:$wb), dir.Oops),
3931        !con(dir.Iops, (ins t_addr_offset_none:$Rn,
3932                            t2am_imm7_offset<memsz.shift>:$addr)),
3933        asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {
3934      bits<4> Rn;
3935      let Inst{18-16} = Rn{2-0};
3936    }
3937  }
3938}
3939
3940multiclass MVE_VLDRSTR_cs_m<MVE_ldst_direction dir, MVE_memsz memsz,
3941                            string asm, string suffix> {
3942  let AM = memsz.AM in {
3943    def "" : MVE_VLDRSTR_cs<
3944        dir, memsz, 1, 0,
3945        dir.Oops, !con(dir.Iops, (ins t2addrmode_imm7<memsz.shift>:$addr)),
3946        asm, suffix, IndexModeNone, "$Qd, $addr", "">;
3947
3948    def _pre : MVE_VLDRSTR_cs<
3949        dir, memsz, 1, 1,
3950        !con((outs rGPR:$wb), dir.Oops),
3951        !con(dir.Iops, (ins t2addrmode_imm7_pre<memsz.shift>:$addr)),
3952        asm, suffix, IndexModePre, "$Qd, $addr!", "$addr.base = $wb"> {
3953      let DecoderMethod = "DecodeMVE_MEM_2_pre<"#memsz.shift#">";
3954    }
3955
3956    def _post : MVE_VLDRSTR_cs<
3957        dir, memsz, 0, 1,
3958        !con((outs rGPR:$wb), dir.Oops),
3959        // We need an !if here to select the base register class,
3960        // because it's legal to write back to SP in a load of this
3961        // type, but not in a store.
3962        !con(dir.Iops, (ins !if(dir.load, t2_addr_offset_none,
3963                                          t2_nosp_addr_offset_none):$Rn,
3964                            t2am_imm7_offset<memsz.shift>:$addr)),
3965        asm, suffix, IndexModePost, "$Qd, $Rn$addr", "$Rn.base = $wb"> {
3966      bits<4> Rn;
3967      let Inst{19-16} = Rn{3-0};
3968    }
3969  }
3970}
3971
3972// Now actually declare all the contiguous load/stores, via those
3973// multiclasses. The instruction ids coming out of this are the bare
3974// names shown in the defm, with _pre or _post appended for writeback,
3975// e.g. MVE_VLDRBS16, MVE_VSTRB16_pre, MVE_VSTRHU16_post.
3976
3977defm MVE_VLDRBS16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s16", 0, 0b01>;
3978defm MVE_VLDRBS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "s32", 0, 0b10>;
3979defm MVE_VLDRBU16: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u16", 1, 0b01>;
3980defm MVE_VLDRBU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memB, "vldrb", "u32", 1, 0b10>;
3981defm MVE_VLDRHS32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "s32", 0, 0b10>;
3982defm MVE_VLDRHU32: MVE_VLDRSTR_cw_m<MVE_ld, MVE_memH, "vldrh", "u32", 1, 0b10>;
3983
3984defm MVE_VLDRBU8:  MVE_VLDRSTR_cs_m<MVE_ld, MVE_memB, "vldrb", "u8">;
3985defm MVE_VLDRHU16: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memH, "vldrh", "u16">;
3986defm MVE_VLDRWU32: MVE_VLDRSTR_cs_m<MVE_ld, MVE_memW, "vldrw", "u32">;
3987
3988defm MVE_VSTRB16:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "16",  0, 0b01>;
3989defm MVE_VSTRB32:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memB, "vstrb", "32",  0, 0b10>;
3990defm MVE_VSTRH32:  MVE_VLDRSTR_cw_m<MVE_st, MVE_memH, "vstrh", "32",  0, 0b10>;
3991
3992defm MVE_VSTRBU8 : MVE_VLDRSTR_cs_m<MVE_st, MVE_memB, "vstrb", "8">;
3993defm MVE_VSTRHU16: MVE_VLDRSTR_cs_m<MVE_st, MVE_memH, "vstrh", "16">;
3994defm MVE_VSTRWU32: MVE_VLDRSTR_cs_m<MVE_st, MVE_memW, "vstrw", "32">;
3995
3996// Gather loads / scatter stores whose address operand is of the form
3997// [Rn,Qm], i.e. a single GPR as the common base address, plus a
3998// vector of offset from it. ('Load/store this sequence of elements of
3999// the same array.')
4000//
4001// Like the contiguous family, these loads and stores can widen the
4002// loaded values / truncate the stored ones, or they can just
4003// load/store the same size of memory and vector lane. But unlike the
4004// contiguous family, there's no particular difference in encoding
4005// between those two cases.
4006//
4007// This family also comes with the option to scale the offset values
4008// in Qm by the size of the loaded memory (i.e. to treat them as array
4009// indices), or not to scale them (to treat them as plain byte offsets
4010// in memory, so that perhaps the loaded values are unaligned). The
4011// scaled instructions' address operand in assembly looks like
4012// [Rn,Qm,UXTW #2] or similar.
4013
4014// Base class.
4015class MVE_VLDRSTR_rq<MVE_ldst_direction dir, MVE_memsz memsz, bit U,
4016                     bits<2> size, bit os, string asm, string suffix, int shift>
4017  : MVE_VLDRSTR_base<dir, U, 0b0, 0b0, 0, dir.Oops,
4018                     !con(dir.Iops, (ins mve_addr_rq_shift<shift>:$addr)),
4019                     asm, suffix, "$Qd, $addr", dir.cstr> {
4020  bits<7> addr;
4021  let Inst{23} = 0b1;
4022  let Inst{19-16} = addr{6-3};
4023  let Inst{8-7} = size;
4024  let Inst{6} = memsz.encoding{1};
4025  let Inst{5} = 0;
4026  let Inst{4} = memsz.encoding{0};
4027  let Inst{3-1} = addr{2-0};
4028  let Inst{0} = os;
4029}
4030
4031// Multiclass that defines the scaled and unscaled versions of an
4032// instruction, when the memory size is wider than a byte. The scaled
4033// version gets the default name like MVE_VLDRBU16_rq; the unscaled /
4034// potentially unaligned version gets a "_u" suffix, e.g.
4035// MVE_VLDRBU16_rq_u.
4036multiclass MVE_VLDRSTR_rq_w<MVE_ldst_direction dir, MVE_memsz memsz,
4037                            string asm, string suffix, bit U, bits<2> size> {
4038  def _u : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;
4039  def "" : MVE_VLDRSTR_rq<dir, memsz, U, size, 1, asm, suffix, memsz.shift>;
4040}
4041
4042// Subclass of MVE_VLDRSTR_rq with the same API as that multiclass,
4043// for use when the memory size is one byte, so there's no 'scaled'
4044// version of the instruction at all. (This is encoded as if it were
4045// unscaled, but named in the default way with no _u suffix.)
4046class MVE_VLDRSTR_rq_b<MVE_ldst_direction dir, MVE_memsz memsz,
4047                       string asm, string suffix, bit U, bits<2> size>
4048  : MVE_VLDRSTR_rq<dir, memsz, U, size, 0, asm, suffix, 0>;
4049
4050// Actually define all the loads and stores in this family.
4051
4052def  MVE_VLDRBU8_rq : MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb","u8",  1,0b00>;
4053def  MVE_VLDRBU16_rq: MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb","u16", 1,0b01>;
4054def  MVE_VLDRBS16_rq: MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb","s16", 0,0b01>;
4055def  MVE_VLDRBU32_rq: MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb","u32", 1,0b10>;
4056def  MVE_VLDRBS32_rq: MVE_VLDRSTR_rq_b<MVE_ld, MVE_memB, "vldrb","s32", 0,0b10>;
4057
4058defm MVE_VLDRHU16_rq: MVE_VLDRSTR_rq_w<MVE_ld, MVE_memH, "vldrh","u16", 1,0b01>;
4059defm MVE_VLDRHU32_rq: MVE_VLDRSTR_rq_w<MVE_ld, MVE_memH, "vldrh","u32", 1,0b10>;
4060defm MVE_VLDRHS32_rq: MVE_VLDRSTR_rq_w<MVE_ld, MVE_memH, "vldrh","s32", 0,0b10>;
4061defm MVE_VLDRWU32_rq: MVE_VLDRSTR_rq_w<MVE_ld, MVE_memW, "vldrw","u32", 1,0b10>;
4062defm MVE_VLDRDU64_rq: MVE_VLDRSTR_rq_w<MVE_ld, MVE_memD, "vldrd","u64", 1,0b11>;
4063
4064def  MVE_VSTRB8_rq  : MVE_VLDRSTR_rq_b<MVE_st, MVE_memB, "vstrb","8",   0,0b00>;
4065def  MVE_VSTRB16_rq : MVE_VLDRSTR_rq_b<MVE_st, MVE_memB, "vstrb","16",  0,0b01>;
4066def  MVE_VSTRB32_rq : MVE_VLDRSTR_rq_b<MVE_st, MVE_memB, "vstrb","32",  0,0b10>;
4067
4068defm MVE_VSTRH16_rq : MVE_VLDRSTR_rq_w<MVE_st, MVE_memH, "vstrh","16",  0,0b01>;
4069defm MVE_VSTRH32_rq : MVE_VLDRSTR_rq_w<MVE_st, MVE_memH, "vstrh","32",  0,0b10>;
4070defm MVE_VSTRW32_rq : MVE_VLDRSTR_rq_w<MVE_st, MVE_memW, "vstrw","32",  0,0b10>;
4071defm MVE_VSTRD64_rq : MVE_VLDRSTR_rq_w<MVE_st, MVE_memD, "vstrd","64",  0,0b11>;
4072
4073// Gather loads / scatter stores whose address operand is of the form
4074// [Qm,#imm], i.e. a vector containing a full base address for each
4075// loaded item, plus an immediate offset applied consistently to all
4076// of them. ('Load/store the same field from this vector of pointers
4077// to a structure type.')
4078//
4079// This family requires the vector lane size to be at least 32 bits
4080// (so there's room for an address in each lane at all). It has no
4081// widening/narrowing variants. But it does support preindex
4082// writeback, in which the address vector is updated to hold the
4083// addresses actually loaded from.
4084
4085// Base class.
4086class MVE_VLDRSTR_qi<MVE_ldst_direction dir, MVE_memsz memsz, bit W, dag wbops,
4087                     string asm, string wbAsm, string suffix, string cstr = "">
4088  : MVE_VLDRSTR_base<dir, 1, 1, W, 1, !con(wbops, dir.Oops),
4089                     !con(dir.Iops, (ins mve_addr_q_shift<memsz.shift>:$addr)),
4090                     asm, suffix, "$Qd, $addr" # wbAsm, cstr # dir.cstr> {
4091  bits<11> addr;
4092  let Inst{23} = addr{7};
4093  let Inst{19-17} = addr{10-8};
4094  let Inst{16} = 0;
4095  let Inst{8} = memsz.encoding{0}; // enough to distinguish 32- from 64-bit
4096  let Inst{7} = 0;
4097  let Inst{6-0} = addr{6-0};
4098}
4099
4100// Multiclass that generates the non-writeback and writeback variants.
4101multiclass MVE_VLDRSTR_qi_m<MVE_ldst_direction dir, MVE_memsz memsz,
4102                            string asm, string suffix> {
4103  def ""   : MVE_VLDRSTR_qi<dir, memsz, 0, (outs),          asm, "",  suffix>;
4104  def _pre : MVE_VLDRSTR_qi<dir, memsz, 1, (outs MQPR:$wb), asm, "!", suffix,
4105                            "$addr.base = $wb"> {
4106    let DecoderMethod="DecodeMVE_MEM_3_pre<"#memsz.shift#">";
4107  }
4108}
4109
4110// Actual instruction definitions.
4111defm MVE_VLDRWU32_qi: MVE_VLDRSTR_qi_m<MVE_ld, MVE_memW, "vldrw", "u32">;
4112defm MVE_VLDRDU64_qi: MVE_VLDRSTR_qi_m<MVE_ld, MVE_memD, "vldrd", "u64">;
4113defm MVE_VSTRW32_qi:  MVE_VLDRSTR_qi_m<MVE_st, MVE_memW, "vstrw", "32">;
4114defm MVE_VSTRD64_qi:  MVE_VLDRSTR_qi_m<MVE_st, MVE_memD, "vstrd", "64">;
4115
4116// Define aliases for all the instructions where memory size and
4117// vector lane size are the same. These are mnemonic aliases, so they
4118// apply consistently across all of the above families - contiguous
4119// loads, and both the rq and qi types of gather/scatter.
4120//
4121// Rationale: As long as you're loading (for example) 16-bit memory
4122// values into 16-bit vector lanes, you can think of them as signed or
4123// unsigned integers, fp16 or just raw 16-bit blobs and it makes no
4124// difference. So we permit all of vldrh.16, vldrh.u16, vldrh.s16,
4125// vldrh.f16 and treat them all as equivalent to the canonical
4126// spelling (which happens to be .u16 for loads, and just .16 for
4127// stores).
4128
4129foreach vpt_cond = ["", "t", "e"] in
4130foreach memsz = [MVE_memB, MVE_memH, MVE_memW, MVE_memD] in
4131foreach suffix = memsz.suffixes in {
4132
4133  // These foreaches are conceptually ifs, implemented by iterating a
4134  // dummy variable over a list with 0 or 1 elements depending on the
4135  // condition. The idea is to iterate over _nearly_ all the suffixes
4136  // in memsz.suffixes, but omit the one we want all the others to alias.
4137
4138  foreach _ = !if(!ne(suffix, memsz.CanonLoadSuffix), [1], []<int>) in
4139  def : MnemonicAlias<
4140    "vldr" # memsz.MnemonicLetter # vpt_cond # suffix,
4141    "vldr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonLoadSuffix>;
4142
4143  foreach _ = !if(!ne(suffix, memsz.CanonStoreSuffix), [1], []<int>) in
4144  def : MnemonicAlias<
4145    "vstr" # memsz.MnemonicLetter # vpt_cond # suffix,
4146    "vstr" # memsz.MnemonicLetter # vpt_cond # memsz.CanonStoreSuffix>;
4147}
4148
4149// end of MVE predicable load/store
4150
4151class MVE_VPT<string suffix, bits<2> size, dag iops, string asm, list<dag> pattern=[]>
4152  : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm, "", pattern> {
4153  bits<3> fc;
4154  bits<4> Mk;
4155  bits<3> Qn;
4156
4157  let Inst{31-23} = 0b111111100;
4158  let Inst{22} = Mk{3};
4159  let Inst{21-20} = size;
4160  let Inst{19-17} = Qn{2-0};
4161  let Inst{16} = 0b1;
4162  let Inst{15-13} = Mk{2-0};
4163  let Inst{12} = fc{2};
4164  let Inst{11-8} = 0b1111;
4165  let Inst{7} = fc{0};
4166  let Inst{4} = 0b0;
4167
4168  let Defs = [VPR, P0];
4169}
4170
4171class MVE_VPTt1<string suffix, bits<2> size, dag iops>
4172  : MVE_VPT<suffix, size, iops, "$fc, $Qn, $Qm"> {
4173  bits<4> Qm;
4174  bits<4> Mk;
4175
4176  let Inst{6} = 0b0;
4177  let Inst{5} = Qm{3};
4178  let Inst{3-1} = Qm{2-0};
4179  let Inst{0} = fc{1};
4180}
4181
4182class MVE_VPTt1i<string suffix, bits<2> size>
4183 : MVE_VPTt1<suffix, size,
4184           (ins vpt_mask:$Mk, pred_basic_i:$fc, MQPR:$Qn, MQPR:$Qm)> {
4185  let Inst{12} = 0b0;
4186  let Inst{0} = 0b0;
4187}
4188
4189def MVE_VPTv4i32 : MVE_VPTt1i<"i32", 0b10>;
4190def MVE_VPTv8i16 : MVE_VPTt1i<"i16", 0b01>;
4191def MVE_VPTv16i8 : MVE_VPTt1i<"i8", 0b00>;
4192
4193class MVE_VPTt1u<string suffix, bits<2> size>
4194 : MVE_VPTt1<suffix, size,
4195           (ins vpt_mask:$Mk, pred_basic_u:$fc, MQPR:$Qn, MQPR:$Qm)> {
4196  let Inst{12} = 0b0;
4197  let Inst{0} = 0b1;
4198}
4199
4200def MVE_VPTv4u32 : MVE_VPTt1u<"u32", 0b10>;
4201def MVE_VPTv8u16 : MVE_VPTt1u<"u16", 0b01>;
4202def MVE_VPTv16u8 : MVE_VPTt1u<"u8", 0b00>;
4203
4204class MVE_VPTt1s<string suffix, bits<2> size>
4205 : MVE_VPTt1<suffix, size,
4206           (ins vpt_mask:$Mk, pred_basic_s:$fc, MQPR:$Qn, MQPR:$Qm)> {
4207  let Inst{12} = 0b1;
4208}
4209
4210def MVE_VPTv4s32 : MVE_VPTt1s<"s32", 0b10>;
4211def MVE_VPTv8s16 : MVE_VPTt1s<"s16", 0b01>;
4212def MVE_VPTv16s8 : MVE_VPTt1s<"s8", 0b00>;
4213
4214class MVE_VPTt2<string suffix, bits<2> size, dag iops>
4215  : MVE_VPT<suffix, size, iops,
4216          "$fc, $Qn, $Rm"> {
4217  bits<4> Rm;
4218  bits<3> fc;
4219  bits<4> Mk;
4220
4221  let Inst{6} = 0b1;
4222  let Inst{5} = fc{1};
4223  let Inst{3-0} = Rm{3-0};
4224}
4225
4226class MVE_VPTt2i<string suffix, bits<2> size>
4227  : MVE_VPTt2<suffix, size,
4228            (ins vpt_mask:$Mk, pred_basic_i:$fc, MQPR:$Qn, GPRwithZR:$Rm)> {
4229  let Inst{12} = 0b0;
4230  let Inst{5} = 0b0;
4231}
4232
4233def MVE_VPTv4i32r : MVE_VPTt2i<"i32", 0b10>;
4234def MVE_VPTv8i16r : MVE_VPTt2i<"i16", 0b01>;
4235def MVE_VPTv16i8r : MVE_VPTt2i<"i8", 0b00>;
4236
4237class MVE_VPTt2u<string suffix, bits<2> size>
4238  : MVE_VPTt2<suffix, size,
4239            (ins vpt_mask:$Mk, pred_basic_u:$fc, MQPR:$Qn, GPRwithZR:$Rm)> {
4240  let Inst{12} = 0b0;
4241  let Inst{5} = 0b1;
4242}
4243
4244def MVE_VPTv4u32r : MVE_VPTt2u<"u32", 0b10>;
4245def MVE_VPTv8u16r : MVE_VPTt2u<"u16", 0b01>;
4246def MVE_VPTv16u8r : MVE_VPTt2u<"u8", 0b00>;
4247
4248class MVE_VPTt2s<string suffix, bits<2> size>
4249  : MVE_VPTt2<suffix, size,
4250            (ins vpt_mask:$Mk, pred_basic_s:$fc, MQPR:$Qn, GPRwithZR:$Rm)> {
4251  let Inst{12} = 0b1;
4252}
4253
4254def MVE_VPTv4s32r : MVE_VPTt2s<"s32", 0b10>;
4255def MVE_VPTv8s16r : MVE_VPTt2s<"s16", 0b01>;
4256def MVE_VPTv16s8r : MVE_VPTt2s<"s8", 0b00>;
4257
4258
4259class MVE_VPTf<string suffix, bit size, dag iops, string asm, list<dag> pattern=[]>
4260  : MVE_MI<(outs ), iops, NoItinerary, !strconcat("vpt", "${Mk}", ".", suffix), asm,
4261            "", pattern> {
4262  bits<3> fc;
4263  bits<4> Mk;
4264  bits<3> Qn;
4265
4266  let Inst{31-29} = 0b111;
4267  let Inst{28} = size;
4268  let Inst{27-23} = 0b11100;
4269  let Inst{22} = Mk{3};
4270  let Inst{21-20} = 0b11;
4271  let Inst{19-17} = Qn{2-0};
4272  let Inst{16} = 0b1;
4273  let Inst{15-13} = Mk{2-0};
4274  let Inst{12} = fc{2};
4275  let Inst{11-8} = 0b1111;
4276  let Inst{7} = fc{0};
4277  let Inst{4} = 0b0;
4278
4279  let Defs = [P0];
4280  let Predicates = [HasMVEFloat];
4281}
4282
4283class MVE_VPTft1<string suffix, bit size>
4284  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, pred_basic_fp:$fc, MQPR:$Qn, MQPR:$Qm),
4285          "$fc, $Qn, $Qm"> {
4286  bits<3> fc;
4287  bits<4> Qm;
4288
4289  let Inst{6} = 0b0;
4290  let Inst{5} = Qm{3};
4291  let Inst{3-1} = Qm{2-0};
4292  let Inst{0} = fc{1};
4293}
4294
4295def MVE_VPTv4f32         : MVE_VPTft1<"f32", 0b0>;
4296def MVE_VPTv8f16         : MVE_VPTft1<"f16", 0b1>;
4297
4298class MVE_VPTft2<string suffix, bit size>
4299  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, pred_basic_fp:$fc, MQPR:$Qn, GPRwithZR:$Rm),
4300          "$fc, $Qn, $Rm"> {
4301  bits<3> fc;
4302  bits<4> Rm;
4303
4304  let Inst{6} = 0b1;
4305  let Inst{5} = fc{1};
4306  let Inst{3-0} = Rm{3-0};
4307}
4308
4309def MVE_VPTv4f32r        : MVE_VPTft2<"f32", 0b0>;
4310def MVE_VPTv8f16r        : MVE_VPTft2<"f16", 0b1>;
4311
4312def MVE_VPST : MVE_MI<(outs ), (ins vpt_mask:$Mk), NoItinerary,
4313       !strconcat("vpst", "${Mk}"), "", "", []> {
4314  bits<4> Mk;
4315
4316  let Inst{31-23} = 0b111111100;
4317  let Inst{22} = Mk{3};
4318  let Inst{21-16} = 0b110001;
4319  let Inst{15-13} = Mk{2-0};
4320  let Inst{12-0} = 0b0111101001101;
4321  let Unpredictable{12} = 0b1;
4322  let Unpredictable{7} = 0b1;
4323  let Unpredictable{5} = 0b1;
4324
4325  let Defs = [P0];
4326}
4327
4328def MVE_VPSEL : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
4329                      "vpsel", "", "$Qd, $Qn, $Qm", vpred_n, "", []> {
4330  bits<4> Qn;
4331  bits<4> Qd;
4332  bits<4> Qm;
4333
4334  let Inst{28} = 0b1;
4335  let Inst{25-23} = 0b100;
4336  let Inst{22} = Qd{3};
4337  let Inst{21-20} = 0b11;
4338  let Inst{19-17} = Qn{2-0};
4339  let Inst{16} = 0b1;
4340  let Inst{15-13} = Qd{2-0};
4341  let Inst{12-9} = 0b0111;
4342  let Inst{8} = 0b1;
4343  let Inst{7} = Qn{3};
4344  let Inst{6} = 0b0;
4345  let Inst{5} = Qm{3};
4346  let Inst{4} = 0b0;
4347  let Inst{3-1} = Qm{2-0};
4348  let Inst{0} = 0b1;
4349}
4350
4351foreach suffix = ["s8", "s16", "s32", "u8", "u16", "u32",
4352                  "i8", "i16", "i32",       "f16", "f32"] in
4353def : MVEInstAlias<"vpsel${vp}." # suffix # "\t$Qd, $Qn, $Qm",
4354                   (MVE_VPSEL MQPR:$Qd, MQPR:$Qn, MQPR:$Qm, vpred_n:$vp)>;
4355
4356def MVE_VPNOT : MVE_p<(outs), (ins), NoItinerary,
4357                      "vpnot", "", "", vpred_n, "", []> {
4358  let Inst{31-0} = 0b11111110001100010000111101001101;
4359  let Unpredictable{19-17} = 0b111;
4360  let Unpredictable{12} = 0b1;
4361  let Unpredictable{7} = 0b1;
4362  let Unpredictable{5} = 0b1;
4363  let Defs = [P0];
4364  let Uses = [P0];
4365
4366  let Constraints = "";
4367}
4368
4369class MVE_loltp_start<dag iops, string asm, string ops, bits<2> size>
4370  : t2LOL<(outs GPRlr:$LR), iops, asm, ops> {
4371  bits<4> Rn;
4372  let Predicates = [HasMVEInt];
4373  let Inst{22} = 0b0;
4374  let Inst{21-20} = size;
4375  let Inst{19-16} = Rn{3-0};
4376  let Inst{12} = 0b0;
4377}
4378
4379class MVE_DLSTP<string asm, bits<2> size>
4380  : MVE_loltp_start<(ins rGPR:$Rn), asm, "$LR, $Rn", size> {
4381  let Inst{13} = 0b1;
4382  let Inst{11-1} = 0b00000000000;
4383  let Unpredictable{10-1} = 0b1111111111;
4384}
4385
4386class MVE_WLSTP<string asm, bits<2> size>
4387  : MVE_loltp_start<(ins rGPR:$Rn, wlslabel_u11:$label),
4388                    asm, "$LR, $Rn, $label", size> {
4389  bits<11> label;
4390  let Inst{13} = 0b0;
4391  let Inst{11} = label{0};
4392  let Inst{10-1} = label{10-1};
4393}
4394
4395def MVE_DLSTP_8  : MVE_DLSTP<"dlstp.8",  0b00>;
4396def MVE_DLSTP_16 : MVE_DLSTP<"dlstp.16", 0b01>;
4397def MVE_DLSTP_32 : MVE_DLSTP<"dlstp.32", 0b10>;
4398def MVE_DLSTP_64 : MVE_DLSTP<"dlstp.64", 0b11>;
4399
4400def MVE_WLSTP_8  : MVE_WLSTP<"wlstp.8",  0b00>;
4401def MVE_WLSTP_16 : MVE_WLSTP<"wlstp.16", 0b01>;
4402def MVE_WLSTP_32 : MVE_WLSTP<"wlstp.32", 0b10>;
4403def MVE_WLSTP_64 : MVE_WLSTP<"wlstp.64", 0b11>;
4404
4405class MVE_loltp_end<dag oops, dag iops, string asm, string ops>
4406  : t2LOL<oops, iops, asm, ops> {
4407  let Predicates = [HasMVEInt];
4408  let Inst{22-21} = 0b00;
4409  let Inst{19-16} = 0b1111;
4410  let Inst{12} = 0b0;
4411}
4412
4413def MVE_LETP : MVE_loltp_end<(outs GPRlr:$LRout),
4414                             (ins GPRlr:$LRin, lelabel_u11:$label),
4415                             "letp", "$LRin, $label"> {
4416  bits<11> label;
4417  let Inst{20} = 0b1;
4418  let Inst{13} = 0b0;
4419  let Inst{11} = label{0};
4420  let Inst{10-1} = label{10-1};
4421}
4422
4423def MVE_LCTP : MVE_loltp_end<(outs), (ins pred:$p), "lctp${p}", ""> {
4424  let Inst{20} = 0b0;
4425  let Inst{13} = 0b1;
4426  let Inst{11-1} = 0b00000000000;
4427  let Unpredictable{21-20} = 0b11;
4428  let Unpredictable{11-1} = 0b11111111111;
4429}
4430
4431
4432//===----------------------------------------------------------------------===//
4433// Patterns
4434//===----------------------------------------------------------------------===//
4435
4436class MVE_unpred_vector_store_typed<ValueType Ty, Instruction RegImmInst,
4437                                    PatFrag StoreKind, int shift>
4438      : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr),
4439           (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr)>;
4440
4441multiclass MVE_unpred_vector_store<Instruction RegImmInst, PatFrag StoreKind,
4442                                   int shift> {
4443  def : MVE_unpred_vector_store_typed<v16i8, RegImmInst, StoreKind, shift>;
4444  def : MVE_unpred_vector_store_typed<v8i16, RegImmInst, StoreKind, shift>;
4445  def : MVE_unpred_vector_store_typed<v8f16, RegImmInst, StoreKind, shift>;
4446  def : MVE_unpred_vector_store_typed<v4i32, RegImmInst, StoreKind, shift>;
4447  def : MVE_unpred_vector_store_typed<v4f32, RegImmInst, StoreKind, shift>;
4448  def : MVE_unpred_vector_store_typed<v2i64, RegImmInst, StoreKind, shift>;
4449  def : MVE_unpred_vector_store_typed<v2f64, RegImmInst, StoreKind, shift>;
4450}
4451
4452class MVE_unpred_vector_load_typed<ValueType Ty, Instruction RegImmInst,
4453                                   PatFrag LoadKind, int shift>
4454      : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr)),
4455          (Ty (RegImmInst t2addrmode_imm7<shift>:$addr))>;
4456
4457multiclass MVE_unpred_vector_load<Instruction RegImmInst, PatFrag LoadKind,
4458                                  int shift> {
4459  def : MVE_unpred_vector_load_typed<v16i8, RegImmInst, LoadKind, shift>;
4460  def : MVE_unpred_vector_load_typed<v8i16, RegImmInst, LoadKind, shift>;
4461  def : MVE_unpred_vector_load_typed<v8f16, RegImmInst, LoadKind, shift>;
4462  def : MVE_unpred_vector_load_typed<v4i32, RegImmInst, LoadKind, shift>;
4463  def : MVE_unpred_vector_load_typed<v4f32, RegImmInst, LoadKind, shift>;
4464  def : MVE_unpred_vector_load_typed<v2i64, RegImmInst, LoadKind, shift>;
4465  def : MVE_unpred_vector_load_typed<v2f64, RegImmInst, LoadKind, shift>;
4466}
4467
4468let Predicates = [HasMVEInt, IsLE] in {
4469  defm : MVE_unpred_vector_store<MVE_VSTRBU8, byte_alignedstore, 0>;
4470  defm : MVE_unpred_vector_store<MVE_VSTRHU16, hword_alignedstore, 1>;
4471  defm : MVE_unpred_vector_store<MVE_VSTRWU32, alignedstore32, 2>;
4472
4473  defm : MVE_unpred_vector_load<MVE_VLDRBU8, byte_alignedload, 0>;
4474  defm : MVE_unpred_vector_load<MVE_VLDRHU16, hword_alignedload, 1>;
4475  defm : MVE_unpred_vector_load<MVE_VLDRWU32, alignedload32, 2>;
4476
4477  def  : Pat<(v16i1 (load t2addrmode_imm7<2>:$addr)),
4478             (v16i1 (VLDR_P0_off t2addrmode_imm7<2>:$addr))>;
4479  def  : Pat<(v8i1 (load t2addrmode_imm7<2>:$addr)),
4480             (v8i1 (VLDR_P0_off t2addrmode_imm7<2>:$addr))>;
4481  def  : Pat<(v4i1 (load t2addrmode_imm7<2>:$addr)),
4482             (v4i1 (VLDR_P0_off t2addrmode_imm7<2>:$addr))>;
4483}
4484
4485let Predicates = [HasMVEInt, IsBE] in {
4486  def : MVE_unpred_vector_store_typed<v16i8, MVE_VSTRBU8, store, 0>;
4487  def : MVE_unpred_vector_store_typed<v8i16, MVE_VSTRHU16, alignedstore16, 1>;
4488  def : MVE_unpred_vector_store_typed<v8f16, MVE_VSTRHU16, alignedstore16, 1>;
4489  def : MVE_unpred_vector_store_typed<v4i32, MVE_VSTRWU32, alignedstore32, 2>;
4490  def : MVE_unpred_vector_store_typed<v4f32, MVE_VSTRWU32, alignedstore32, 2>;
4491
4492  def : MVE_unpred_vector_load_typed<v16i8, MVE_VLDRBU8, load, 0>;
4493  def : MVE_unpred_vector_load_typed<v8i16, MVE_VLDRHU16, alignedload16, 1>;
4494  def : MVE_unpred_vector_load_typed<v8f16, MVE_VLDRHU16, alignedload16, 1>;
4495  def : MVE_unpred_vector_load_typed<v4i32, MVE_VLDRWU32, alignedload32, 2>;
4496  def : MVE_unpred_vector_load_typed<v4f32, MVE_VLDRWU32, alignedload32, 2>;
4497}
4498
4499
4500// Widening/Narrowing Loads/Stores
4501
4502let Predicates = [HasMVEInt] in {
4503  def : Pat<(truncstorevi8  (v8i16 MQPR:$val), t2addrmode_imm7<1>:$addr),
4504             (MVE_VSTRB16 MQPR:$val, t2addrmode_imm7<1>:$addr)>;
4505  def : Pat<(truncstorevi8  (v4i32 MQPR:$val), t2addrmode_imm7<1>:$addr),
4506             (MVE_VSTRB32 MQPR:$val, t2addrmode_imm7<1>:$addr)>;
4507  def : Pat<(truncstorevi16 (v4i32 MQPR:$val), t2addrmode_imm7<2>:$addr),
4508             (MVE_VSTRH32 MQPR:$val, t2addrmode_imm7<2>:$addr)>;
4509}
4510
4511multiclass MVEExtLoad<string DestLanes, string DestElemBits,
4512                      string SrcElemBits, string SrcElemType,
4513                      Operand am> {
4514  def _Any : Pat<(!cast<ValueType>("v" # DestLanes # "i" # DestElemBits)
4515                   (!cast<PatFrag>("extloadvi"  # SrcElemBits) am:$addr)),
4516                 (!cast<Instruction>("MVE_VLDR" # SrcElemType # "U" # DestElemBits)
4517                   am:$addr)>;
4518  def _Z   : Pat<(!cast<ValueType>("v" # DestLanes # "i" # DestElemBits)
4519                   (!cast<PatFrag>("zextloadvi"  # SrcElemBits) am:$addr)),
4520                 (!cast<Instruction>("MVE_VLDR" # SrcElemType # "U" # DestElemBits)
4521                   am:$addr)>;
4522  def _S   : Pat<(!cast<ValueType>("v" # DestLanes # "i" # DestElemBits)
4523                   (!cast<PatFrag>("sextloadvi"  # SrcElemBits) am:$addr)),
4524                 (!cast<Instruction>("MVE_VLDR" # SrcElemType # "S" # DestElemBits)
4525                   am:$addr)>;
4526}
4527
4528let Predicates = [HasMVEInt] in {
4529  defm : MVEExtLoad<"4", "32", "8",  "B", t2addrmode_imm7<1>>;
4530  defm : MVEExtLoad<"8", "16", "8",  "B", t2addrmode_imm7<1>>;
4531  defm : MVEExtLoad<"4", "32", "16", "H", t2addrmode_imm7<2>>;
4532}
4533
4534
4535// Bit convert patterns
4536
4537let Predicates = [HasMVEInt] in {
4538  def : Pat<(v2f64 (bitconvert (v2i64 QPR:$src))), (v2f64 QPR:$src)>;
4539  def : Pat<(v2i64 (bitconvert (v2f64 QPR:$src))), (v2i64 QPR:$src)>;
4540
4541  def : Pat<(v4i32 (bitconvert (v4f32 QPR:$src))), (v4i32 QPR:$src)>;
4542  def : Pat<(v4f32 (bitconvert (v4i32 QPR:$src))), (v4f32 QPR:$src)>;
4543
4544  def : Pat<(v8i16 (bitconvert (v8f16 QPR:$src))), (v8i16  QPR:$src)>;
4545  def : Pat<(v8f16 (bitconvert (v8i16 QPR:$src))), (v8f16  QPR:$src)>;
4546}
4547
4548let Predicates = [IsLE,HasMVEInt] in {
4549  def : Pat<(v2f64 (bitconvert (v4f32 QPR:$src))), (v2f64 QPR:$src)>;
4550  def : Pat<(v2f64 (bitconvert (v4i32 QPR:$src))), (v2f64 QPR:$src)>;
4551  def : Pat<(v2f64 (bitconvert (v8f16 QPR:$src))), (v2f64 QPR:$src)>;
4552  def : Pat<(v2f64 (bitconvert (v8i16 QPR:$src))), (v2f64 QPR:$src)>;
4553  def : Pat<(v2f64 (bitconvert (v16i8 QPR:$src))), (v2f64 QPR:$src)>;
4554
4555  def : Pat<(v2i64 (bitconvert (v4f32 QPR:$src))), (v2i64 QPR:$src)>;
4556  def : Pat<(v2i64 (bitconvert (v4i32 QPR:$src))), (v2i64 QPR:$src)>;
4557  def : Pat<(v2i64 (bitconvert (v8f16 QPR:$src))), (v2i64 QPR:$src)>;
4558  def : Pat<(v2i64 (bitconvert (v8i16 QPR:$src))), (v2i64 QPR:$src)>;
4559  def : Pat<(v2i64 (bitconvert (v16i8 QPR:$src))), (v2i64 QPR:$src)>;
4560
4561  def : Pat<(v4f32 (bitconvert (v2f64 QPR:$src))), (v4f32 QPR:$src)>;
4562  def : Pat<(v4f32 (bitconvert (v2i64 QPR:$src))), (v4f32 QPR:$src)>;
4563  def : Pat<(v4f32 (bitconvert (v8f16 QPR:$src))), (v4f32 QPR:$src)>;
4564  def : Pat<(v4f32 (bitconvert (v8i16 QPR:$src))), (v4f32 QPR:$src)>;
4565  def : Pat<(v4f32 (bitconvert (v16i8 QPR:$src))), (v4f32 QPR:$src)>;
4566
4567  def : Pat<(v4i32 (bitconvert (v2f64 QPR:$src))), (v4i32 QPR:$src)>;
4568  def : Pat<(v4i32 (bitconvert (v2i64 QPR:$src))), (v4i32 QPR:$src)>;
4569  def : Pat<(v4i32 (bitconvert (v8f16 QPR:$src))), (v4i32 QPR:$src)>;
4570  def : Pat<(v4i32 (bitconvert (v8i16 QPR:$src))), (v4i32 QPR:$src)>;
4571  def : Pat<(v4i32 (bitconvert (v16i8 QPR:$src))), (v4i32 QPR:$src)>;
4572
4573  def : Pat<(v8f16 (bitconvert (v2f64 QPR:$src))), (v8f16 QPR:$src)>;
4574  def : Pat<(v8f16 (bitconvert (v2i64 QPR:$src))), (v8f16 QPR:$src)>;
4575  def : Pat<(v8f16 (bitconvert (v4f32 QPR:$src))), (v8f16 QPR:$src)>;
4576  def : Pat<(v8f16 (bitconvert (v4i32 QPR:$src))), (v8f16 QPR:$src)>;
4577  def : Pat<(v8f16 (bitconvert (v16i8 QPR:$src))), (v8f16 QPR:$src)>;
4578
4579  def : Pat<(v8i16 (bitconvert (v2f64 QPR:$src))), (v8i16 QPR:$src)>;
4580  def : Pat<(v8i16 (bitconvert (v2i64 QPR:$src))), (v8i16 QPR:$src)>;
4581  def : Pat<(v8i16 (bitconvert (v4f32 QPR:$src))), (v8i16 QPR:$src)>;
4582  def : Pat<(v8i16 (bitconvert (v4i32 QPR:$src))), (v8i16 QPR:$src)>;
4583  def : Pat<(v8i16 (bitconvert (v16i8 QPR:$src))), (v8i16 QPR:$src)>;
4584
4585  def : Pat<(v16i8 (bitconvert (v2f64 QPR:$src))), (v16i8 QPR:$src)>;
4586  def : Pat<(v16i8 (bitconvert (v2i64 QPR:$src))), (v16i8 QPR:$src)>;
4587  def : Pat<(v16i8 (bitconvert (v4f32 QPR:$src))), (v16i8 QPR:$src)>;
4588  def : Pat<(v16i8 (bitconvert (v4i32 QPR:$src))), (v16i8 QPR:$src)>;
4589  def : Pat<(v16i8 (bitconvert (v8f16 QPR:$src))), (v16i8 QPR:$src)>;
4590  def : Pat<(v16i8 (bitconvert (v8i16 QPR:$src))), (v16i8 QPR:$src)>;
4591}
4592