Lines Matching refs:VTI
314 multiclass MVE_TwoOpPattern<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,
318 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
319 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
322 if !ne(VTI.Size, 0b11) then {
323 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
324 (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
325 (VTI.Vec MQPR:$Qn))),
326 (VTI.Vec MQPR:$inactive))),
327 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
328 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
329 (VTI.Vec MQPR:$inactive)))>;
332 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
333 (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
334 (VTI.Vec MQPR:$Qn),
335 (VTI.Vec IdentityVec))))),
336 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
337 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
338 (VTI.Vec MQPR:$Qm)))>;
342 def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)),
344 (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
345 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
346 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
347 (VTI.Vec MQPR:$inactive)))>;
350 multiclass MVE_TwoOpPatternDup<MVEVectorVTInfo VTI, SDPatternOperator Op, Intrinsic PredInt,
354 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn)))),
355 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn))>;
358 if !ne(VTI.Size, 0b11) then {
359 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
360 (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
361 (VTI.Vec (ARMvdup rGPR:$Rn)))),
362 (VTI.Vec MQPR:$inactive))),
363 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
364 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
365 (VTI.Vec MQPR:$inactive)))>;
368 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
369 (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
371 (VTI.Vec IdentityVec))))),
372 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
373 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
374 (VTI.Vec MQPR:$Qm)))>;
378 def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))),
380 (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
381 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
382 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
383 (VTI.Vec MQPR:$inactive)))>;
630 multiclass MVE_VABAV_m<MVEVectorVTInfo VTI> {
631 def "" : MVE_VABAV<VTI.Suffix, VTI.Unsigned, VTI.Size>;
636 (i32 VTI.Unsigned),
638 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
640 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
643 (i32 VTI.Unsigned),
645 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
646 (VTI.Pred VCCR:$mask))),
648 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
649 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
689 multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
690 def acc : MVE_VADDV<"vaddva", VTI.Suffix,
692 0b1, VTI.Unsigned, VTI.Size>;
693 def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,
695 0b0, VTI.Unsigned, VTI.Size>;
701 if VTI.Unsigned then {
702 def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
704 def : Pat<(i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
705 (VTI.Vec MQPR:$vec),
706 (VTI.Vec ARMimmAllZerosV))))),
708 def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
710 def : Pat<(i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
712 def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
715 def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
716 (VTI.Vec MQPR:$vec),
717 (VTI.Vec ARMimmAllZerosV))))),
720 def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
723 def : Pat<(i32 (add (i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
727 def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
729 def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
732 def : Pat<(i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
734 def : Pat<(i32 (add (i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
739 def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
740 (i32 VTI.Unsigned),
741 (VTI.Pred VCCR:$pred))),
743 def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
744 (i32 VTI.Unsigned),
745 (VTI.Pred VCCR:$pred)),
794 multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {
795 def acc : MVE_VADDLV<"vaddlva", VTI.Suffix,
798 0b1, VTI.Unsigned>;
799 def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,
801 0b0, VTI.Unsigned>;
806 defvar letter = VTI.SuffixLetter;
817 def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),
818 (InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;
820 (VTI.Pred VCCR:$pred)),
822 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg)>;
855 MVEVectorVTInfo VTI, string intrBaseName,
857 def "": MVE_VMINMAXNMV<iname, VTI.Suffix, VTI.Size{0}, notAbs, isMin>;
864 (VTI.Vec MQPR:$vec))),
866 (VTI.Vec MQPR:$vec)),
869 (VTI.Vec MQPR:$vec),
870 (VTI.Pred VCCR:$pred))),
872 (VTI.Vec MQPR:$vec),
873 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg),
913 MVEVectorVTInfo VTI, string intrBaseName> {
914 def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
919 defvar base_args = (? (i32 rGPR:$prev), (VTI.Vec MQPR:$vec));
920 defvar args = !if(notAbs, !con(base_args, (? (i32 VTI.Unsigned))),
925 (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec)))>;
926 def : Pat<(i32 !con(args, (pred_intr (VTI.Pred VCCR:$pred)))),
927 (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec),
928 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
1044 multiclass MVE_VMLAMLSDAV_A<string iname, string x, MVEVectorVTInfo VTI,
1046 def ""#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # x, VTI.Suffix,
1048 sz, bit_28, 0b0, X, bit_8, bit_0, VTI.Size>;
1049 def "a"#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # "a" # x, VTI.Suffix,
1052 sz, bit_28, 0b1, X, bit_8, bit_0, VTI.Size>;
1055 (i32 VTI.Unsigned),
1059 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
1060 (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
1061 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
1064 (i32 VTI.Unsigned),
1068 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1069 (VTI.Pred VCCR:$mask))),
1070 (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
1071 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1072 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
1075 (i32 VTI.Unsigned),
1079 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
1080 (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
1082 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
1085 (i32 VTI.Unsigned),
1089 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1090 (VTI.Pred VCCR:$mask))),
1091 (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
1093 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1094 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
1098 multiclass MVE_VMLAMLSDAV_AX<string iname, MVEVectorVTInfo VTI, bit sz,
1100 defm "" : MVE_VMLAMLSDAV_A<iname, "", VTI, sz, bit_28,
1102 defm "" : MVE_VMLAMLSDAV_A<iname, "x", VTI, sz, bit_28,
1114 multiclass MVE_VMLSDAV_multi<MVEVectorVTInfo VTI, bit sz, bit bit_28> {
1115 defm "" : MVE_VMLAMLSDAV_AX<"vmlsdav", VTI,
1304 multiclass MVE_VRMLALDAVH_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {
1305 defm "" : MVE_VMLALDAVBase_AX<"vrmlaldavh", "s"#VTI.BitsSuffix,
1306 0b0, 0b0, 0b1, 0b0, VTI.Size, pattern>;
1307 defm "" : MVE_VMLALDAVBase_A<"vrmlaldavh", "", "u"#VTI.BitsSuffix,
1308 0b0, 0b1, 0b0, 0b1, 0b0, VTI.Size, pattern>;
1331 multiclass MVE_VMLALDAV_multi<MVEVectorVTInfo VTI, list<dag> pattern=[]> {
1332 defm "" : MVE_VMLALDAVBase_AX<"vmlaldav", "s"#VTI.BitsSuffix,
1333 VTI.Size{1}, 0b0, 0b0, 0b0, VTI.Size, pattern>;
1334 defm "" : MVE_VMLALDAVBase_A<"vmlaldav", "", "u"#VTI.BitsSuffix,
1335 VTI.Size{1}, 0b1, 0b0, 0b0, 0b0, VTI.Size, pattern>;
1442 multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredInt> {
1443 def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size, bit_4>;
1446 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 0)), !cast<Instruction>(NAME)>;
1471 multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,
1473 def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;
1476 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
1480 multiclass MVE_VMAX<MVEVectorVTInfo VTI>
1481 : MVE_VMINMAX_m<"vmax", 0b0, VTI, !if(VTI.Unsigned, umax, smax), int_arm_mve_max_predicated>;
1482 multiclass MVE_VMIN<MVEVectorVTInfo VTI>
1483 : MVE_VMINMAX_m<"vmin", 0b1, VTI, !if(VTI.Unsigned, umin, smin), int_arm_mve_min_predicated>;
1569 foreach VTI = VTIs in {
1570 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src))),
1571 (VTI.Vec (Inst (VTI.Vec MQPR:$src)))>;
1572 def : Pat<(VTI.Vec (int_arm_mve_vrev_predicated (VTI.Vec MQPR:$src),
1573 revbits, (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1574 (VTI.Vec (Inst (VTI.Vec MQPR:$src), ARMVCCThen,
1575 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
1602 foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in {
1603 def : Pat<(VTI.Vec (vnotq (VTI.Vec MQPR:$val1))),
1604 (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1)))>;
1605 def : Pat<(VTI.Vec (int_arm_mve_mvn_predicated (VTI.Vec MQPR:$val1),
1606 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1607 (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1), ARMVCCThen,
1608 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
1708 MVEVectorVTInfo VTI, Operand imm_type, SDNode op> {
1709 def "" : MVE_bit_cmode<iname, VTI.Suffix, VTI.Size{0},
1710 (ins MQPR:$Qd_src, imm_type:$imm), VTI.Size> {
1716 defvar UnpredPat = (VTI.Vec (op (VTI.Vec MQPR:$src), timm:$simm));
1720 (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm))>;
1721 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
1722 UnpredPat, (VTI.Vec MQPR:$src))),
1723 (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm,
1724 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
1728 multiclass MVE_VORRimm<MVEVectorVTInfo VTI, Operand imm_type> {
1729 defm "": MVE_bit_cmode_p<"vorr", 0, VTI, imm_type, ARMvorrImm>;
1731 multiclass MVE_VBICimm<MVEVectorVTInfo VTI, Operand imm_type> {
1732 defm "": MVE_bit_cmode_p<"vbic", 1, VTI, imm_type, ARMvbicImm>;
1972 multiclass MVE_VMUL_m<MVEVectorVTInfo VTI> {
1973 def "" : MVE_VMULt1<"vmul", VTI.Suffix, VTI.Size>;
1976 defm : MVE_TwoOpPattern<VTI, mul, int_arm_mve_mul_predicated, (? ),
2000 multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,
2003 def "" : MVE_VQxDMULH_Base<iname, VTI.Suffix, VTI.Size, rounding>;
2007 defm : MVE_TwoOpPattern<VTI, Op, pred_int, (? ), Inst>;
2010 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
2011 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2015 multiclass MVE_VQxDMULH<string iname, MVEVectorVTInfo VTI, bit rounding>
2016 : MVE_VQxDMULH_m<iname, VTI, !if(rounding, null_frag,
2045 multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,
2047 def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;
2051 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
2055 multiclass MVE_VADD<MVEVectorVTInfo VTI>
2056 : MVE_VADDSUB_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
2057 multiclass MVE_VSUB<MVEVectorVTInfo VTI>
2058 : MVE_VADDSUB_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
2088 multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,
2090 def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2094 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2099 multiclass MVE_VQADD<MVEVectorVTInfo VTI, SDNode unpred_op>
2100 : MVE_VQADD_m<VTI, unpred_op, int_arm_mve_qadd_predicated>;
2109 multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,
2111 def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2115 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2120 multiclass MVE_VQSUB<MVEVectorVTInfo VTI, SDNode unpred_op>
2121 : MVE_VQSUB_m<VTI, unpred_op, int_arm_mve_qsub_predicated>;
2143 multiclass MVE_VABD_m<MVEVectorVTInfo VTI, SDNode Op,
2145 def "" : MVE_VABD_int<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2149 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2153 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2154 (i32 VTI.Unsigned))),
2155 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2159 multiclass MVE_VABD<MVEVectorVTInfo VTI, SDNode Op>
2160 : MVE_VABD_m<VTI, Op, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
2201 multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI, SDNode Op,
2203 def "" : MVE_VRHADD_Base<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2205 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
2209 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2210 (i32 VTI.Unsigned))),
2211 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2215 multiclass MVE_VRHADD<MVEVectorVTInfo VTI, SDNode rhadd>
2216 : MVE_VRHADD_m<VTI, rhadd, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;
2305 multiclass MVE_VHADD_m<MVEVectorVTInfo VTI, SDNode Op,
2308 def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2310 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
2314 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),
2315 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2317 def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2322 multiclass MVE_VHADD<MVEVectorVTInfo VTI, SDNode Op, PatFrag add_op, SDNode shift_op>
2323 : MVE_VHADD_m<VTI, Op, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, add_op,
2337 multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,
2340 def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2345 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2346 (i32 VTI.Unsigned))),
2347 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2349 def : Pat<(VTI.Vec (shift_op (sub_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2354 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2355 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2356 (VTI.Vec MQPR:$inactive))),
2357 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2358 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
2359 (VTI.Vec MQPR:$inactive)))>;
2363 multiclass MVE_VHSUB<MVEVectorVTInfo VTI, PatFrag sub_op, SDNode shift_op>
2364 : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, sub_op,
2471 multiclass MVE_VCLSCLZ_p<string opname, bit opcode, MVEVectorVTInfo VTI,
2473 def "": MVE_VCLSCLZ<"v"#opname, VTI.Suffix, VTI.Size, opcode>;
2479 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
2480 (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
2481 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
2482 (VTI.Vec MQPR:$inactive))),
2483 (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
2484 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
2517 MVEVectorVTInfo VTI> {
2518 def "" : MVE_VABSNEG_int<iname, VTI.Suffix, VTI.Size, negate, saturate>;
2524 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),
2525 (VTI.Vec (Inst $v))>;
2528 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
2529 (VTI.Vec MQPR:$inactive))),
2530 (VTI.Vec (Inst $v, ARMVCCThen, $mask, zero_reg, $inactive))>;
2534 foreach VTI = [ MVE_v16s8, MVE_v8s16, MVE_v4s32 ] in {
2535 defm "MVE_VABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2536 "vabs", 0, 0, abs, int_arm_mve_abs_predicated, VTI>;
2537 defm "MVE_VQABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2538 "vqabs", 0, 1, ?, int_arm_mve_qabs_predicated, VTI>;
2539 defm "MVE_VNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2540 "vneg", 1, 0, vnegq, int_arm_mve_neg_predicated, VTI>;
2541 defm "MVE_VQNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2542 "vqneg", 1, 1, ?, int_arm_mve_qneg_predicated, VTI>;
2545 // int_min/int_max: vector containing INT_MIN/INT_MAX VTI.Size times
2547 multiclass vqabsneg_pattern<MVEVectorVTInfo VTI, dag int_min, dag int_max,
2554 def : Pat<(VTI.Vec (vselect
2555 (VTI.Pred (ARMvcmpz (VTI.Vec MQPR:$reg), ARMCCgt)),
2556 (VTI.Vec MQPR:$reg),
2557 (VTI.Vec (vselect
2558 (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2560 (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))))),
2561 (VTI.Vec (vqabs_instruction (VTI.Vec MQPR:$reg)))>;
2564 def : Pat<(VTI.Vec (vselect
2565 (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2567 (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))),
2568 (VTI.Vec (vqneg_instruction (VTI.Vec MQPR:$reg)))>;
2685 multiclass MVE_VMINMAXA_m<string iname, MVEVectorVTInfo VTI,
2687 def "" : MVE_VMINMAXA<iname, VTI.Suffix, VTI.Size, bit_12>;
2692 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qd), (abs (VTI.Vec MQPR:$Qm)))),
2693 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
2696 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2697 (VTI.Pred VCCR:$mask))),
2698 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2699 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
2703 multiclass MVE_VMINA<MVEVectorVTInfo VTI>
2704 : MVE_VMINMAXA_m<"vmina", VTI, umin, int_arm_mve_vmina_predicated, 0b1>;
2710 multiclass MVE_VMAXA<MVEVectorVTInfo VTI>
2711 : MVE_VMINMAXA_m<"vmaxa", VTI, umax, int_arm_mve_vmaxa_predicated, 0b0>;
2898 multiclass MVE_VSHLL_patterns<MVEVectorVTInfo VTI, int top> {
2899 defvar suffix = !strconcat(VTI.Suffix, !if(top, "th", "bh"));
2906 def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), imm:$imm,
2907 (i32 VTI.Unsigned), (i32 top))),
2908 (VTI.DblVec (inst_imm (VTI.Vec MQPR:$src), imm:$imm))>;
2909 def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2910 (i32 VTI.Unsigned), (i32 top))),
2911 (VTI.DblVec (inst_lw (VTI.Vec MQPR:$src)))>;
2913 def : Pat<(VTI.DblVec (pred_int (VTI.Vec MQPR:$src), imm:$imm,
2914 (i32 VTI.Unsigned), (i32 top),
2915 (VTI.DblPred VCCR:$mask),
2916 (VTI.DblVec MQPR:$inactive))),
2917 (VTI.DblVec (inst_imm (VTI.Vec MQPR:$src), imm:$imm,
2918 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
2919 (VTI.DblVec MQPR:$inactive)))>;
2920 def : Pat<(VTI.DblVec (pred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2921 (i32 VTI.Unsigned), (i32 top),
2922 (VTI.DblPred VCCR:$mask),
2923 (VTI.DblVec MQPR:$inactive))),
2924 (VTI.DblVec (inst_lw (VTI.Vec MQPR:$src), ARMVCCThen,
2925 (VTI.DblPred VCCR:$mask), zero_reg,
2926 (VTI.DblVec MQPR:$inactive)))>;
2929 foreach VTI = [MVE_v16s8, MVE_v8s16, MVE_v16u8, MVE_v8u16] in
2931 defm : MVE_VSHLL_patterns<VTI, top>;
3162 multiclass MVE_shift_by_vec_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
3163 def "" : MVE_shift_by_vec<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
3166 def : Pat<(VTI.Vec (int_arm_mve_vshl_vector
3167 (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3168 (i32 q), (i32 r), (i32 VTI.Unsigned))),
3169 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh)))>;
3171 def : Pat<(VTI.Vec (int_arm_mve_vshl_vector_predicated
3172 (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3173 (i32 q), (i32 r), (i32 VTI.Unsigned),
3174 (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
3175 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3176 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3177 (VTI.Vec MQPR:$inactive)))>;
3228 MVEVectorVTInfo VTI;
3275 MVEVectorVTInfo VTI> {
3276 defvar inparams = (? (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3278 defvar outparams = (inst (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3283 def : Pat<(VTI.Vec !setdagop(inparams, unpred_int)),
3284 (VTI.Vec outparams)>;
3285 def : Pat<(VTI.Vec !con(inparams, (pred_int (VTI.Pred VCCR:$pred)))),
3286 (VTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred, zero_reg)))>;
3307 let VTI = VTI_;
3309 let unsignedFlag = (? (i32 VTI.Unsigned));
3347 let VTI = VTI_;
3377 let VTI = VTI_;
3379 let unsignedFlag = (? (i32 VTI.Unsigned));
3410 def : Pat<(inst.VTI.Vec !con((inst.unpred_int (inst.VTI.Vec MQPR:$src),
3413 (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3416 def : Pat<(inst.VTI.Vec !con((inst.pred_int (inst.VTI.Vec MQPR:$src),
3419 (? (inst.VTI.Pred VCCR:$mask),
3420 (inst.VTI.Vec MQPR:$inactive)))),
3421 (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3423 ARMVCCThen, (inst.VTI.Pred VCCR:$mask), zero_reg,
3424 (inst.VTI.Vec MQPR:$inactive)))>;
3509 MVEVectorVTInfo VTI, Operand imm_operand_type, SDNode unpred_op,
3512 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src), imm_operand_type:$imm)),
3513 (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm))>;
3515 def : Pat<(VTI.Vec !con((pred_int (VTI.Vec MQPR:$src), imm_operand_type:$imm),
3517 (pred_int (VTI.Pred VCCR:$mask),
3518 (VTI.Vec MQPR:$inactive)))),
3519 (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm,
3520 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3521 (VTI.Vec MQPR:$inactive)))>;
3524 multiclass MVE_immediate_shift_patterns<MVEVectorVTInfo VTI,
3526 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3528 !cast<Instruction>("MVE_VSHL_immi" # VTI.BitsSuffix)>;
3529 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3531 !cast<Instruction>("MVE_VSHR_immu" # VTI.BitsSuffix), [1]>;
3532 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3534 !cast<Instruction>("MVE_VSHR_imms" # VTI.BitsSuffix), [0]>;
3579 multiclass MVE_VRINT_m<MVEVectorVTInfo VTI, string suffix, bits<3> opcode,
3581 def "": MVE_VRINT<suffix, opcode, VTI.Suffix, VTI.Size>;
3586 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
3587 (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
3588 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
3589 (VTI.Vec MQPR:$inactive))),
3590 (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
3591 (VTI.Pred VCCR:$pred), zero_reg, (VTI.Vec MQPR:$inactive)))>;
3595 multiclass MVE_VRINT_ops<MVEVectorVTInfo VTI> {
3596 defm N : MVE_VRINT_m<VTI, "n", 0b000, int_arm_mve_vrintn>;
3597 defm X : MVE_VRINT_m<VTI, "x", 0b001, frint>;
3598 defm A : MVE_VRINT_m<VTI, "a", 0b010, fround>;
3599 defm Z : MVE_VRINT_m<VTI, "z", 0b011, ftrunc>;
3600 defm M : MVE_VRINT_m<VTI, "m", 0b101, ffloor>;
3601 defm P : MVE_VRINT_m<VTI, "p", 0b111, fceil>;
3634 multiclass MVE_VMULT_fp_m<string iname, MVEVectorVTInfo VTI, SDNode Op,
3636 def "" : MVE_VMUL_fp<iname, VTI.Suffix, VTI.Size>;
3640 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), IdentityVec>;
3644 multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>
3645 : MVE_VMULT_fp_m<"vmul", VTI, fmul, int_arm_mve_mul_predicated, IdentityVec>;
3673 multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI> {
3674 def "" : MVE_VCMLA<VTI.Suffix, VTI.Size>;
3678 def : Pat<(VTI.Vec (int_arm_mve_vcmlaq
3679 imm:$rot, (VTI.Vec MQPR:$Qd_src),
3680 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3681 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
3682 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3685 def: Pat<(VTI.Vec (fadd_contract MQPR:$Qd_src,
3687 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))),
3688 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
3689 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3692 def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated
3693 imm:$rot, (VTI.Vec MQPR:$Qd_src),
3694 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3695 (VTI.Pred VCCR:$mask))),
3696 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),
3697 (VTI.Vec MQPR:$Qm), imm:$rot,
3698 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
3728 multiclass MVE_VFMA_fp_multi<string iname, bit fms, MVEVectorVTInfo VTI> {
3729 def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size, 0b1, 0b0, fms,
3733 defvar m1 = (VTI.Vec MQPR:$m1);
3734 defvar m2 = (VTI.Vec MQPR:$m2);
3735 defvar add = (VTI.Vec MQPR:$add);
3736 defvar pred = (VTI.Pred VCCR:$pred);
3740 def : Pat<(VTI.Vec (fma (fneg m1), m2, add)),
3742 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
3743 (VTI.Vec (fma (fneg m1), m2, add)),
3746 def : Pat<(VTI.Vec (pred_int (fneg m1), m2, add, pred)),
3748 def : Pat<(VTI.Vec (pred_int m1, (fneg m2), add, pred)),
3751 def : Pat<(VTI.Vec (fma m1, m2, add)),
3753 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
3754 (VTI.Vec (fma m1, m2, add)),
3757 def : Pat<(VTI.Vec (pred_int m1, m2, add, pred)),
3768 multiclass MVE_VADDSUB_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
3770 def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size, 0, 1, bit_21> {
3776 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), IdentityVec>;
3780 multiclass MVE_VADD_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>
3781 : MVE_VADDSUB_fp_m<"vadd", 0, VTI, fadd, int_arm_mve_add_predicated, IdentityVec>;
3782 multiclass MVE_VSUB_fp_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec>
3783 : MVE_VADDSUB_fp_m<"vsub", 1, VTI, fsub, int_arm_mve_sub_predicated, IdentityVec>;
3815 multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, string cstr=""> {
3816 def "" : MVE_VCADD<VTI.Suffix, VTI.Size, cstr>;
3820 def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),
3821 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3822 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3825 def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),
3826 imm:$rot, (VTI.Vec MQPR:$inactive),
3827 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3828 (VTI.Pred VCCR:$mask))),
3829 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3830 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3831 (VTI.Vec MQPR:$inactive)))>;
3859 multiclass MVE_VABDT_fp_m<MVEVectorVTInfo VTI,
3861 def "" : MVE_VABD_fp<VTI.Suffix, VTI.Size>;
3865 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3867 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
3868 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3869 (i32 0), (VTI.Pred VCCR:$mask),
3870 (VTI.Vec MQPR:$inactive))),
3871 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3872 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
3873 (VTI.Vec MQPR:$inactive)))>;
3877 multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>
3878 : MVE_VABDT_fp_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
4114 MVEVectorVTInfo VTI, bit opcode> {
4115 def "" : MVE_VABSNEG_fp<iname, VTI.Suffix, VTI.Size, opcode>;
4119 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),
4120 (VTI.Vec (Inst $v))>;
4121 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
4122 (VTI.Vec MQPR:$inactive))),
4123 (VTI.Vec (Inst $v, ARMVCCThen, $mask, zero_reg, $inactive))>;
4160 multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,
4163 def "" : MVE_VMAXMINNMA<iname, VTI.Suffix, VTI.Size, bit_12>;
4168 def : Pat<(VTI.Vec (unpred_op (fabs (VTI.Vec MQPR:$Qd)),
4169 (fabs (VTI.Vec MQPR:$Qm)))),
4170 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
4173 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
4174 (VTI.Pred VCCR:$mask))),
4175 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
4176 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
4180 multiclass MVE_VMAXNMA<MVEVectorVTInfo VTI, bit bit_12>
4181 : MVE_VMAXMINNMA_m<"vmaxnma", VTI, fmaxnum, int_arm_mve_vmaxnma_predicated, bit_12>;
4186 multiclass MVE_VMINNMA<MVEVectorVTInfo VTI, bit bit_12>
4187 : MVE_VMAXMINNMA_m<"vminnma", VTI, fminnum, int_arm_mve_vminnma_predicated, bit_12>;
4564 MVEVectorVTInfo VTI> {
4565 def "": MVE_VQxDMLxDH<iname, exch, round, subtract, VTI.Suffix, VTI.Size,
4566 !if(!eq(VTI.LaneBits, 32), ",@earlyclobber $Qd", "")>;
4572 def : Pat<(VTI.Vec !con((unpred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4573 (VTI.Vec MQPR:$c)), ConstParams)),
4574 (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4575 (VTI.Vec MQPR:$c)))>;
4576 def : Pat<(VTI.Vec !con((pred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4577 (VTI.Vec MQPR:$c)), ConstParams,
4578 (? (VTI.Pred VCCR:$pred)))),
4579 (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4580 (VTI.Vec MQPR:$c),
4581 ARMVCCThen, (VTI.Pred VCCR:$pred), zero_reg))>;
4620 multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,
4622 def "" : MVE_VCMUL<iname, VTI.Suffix, VTI.Size, cstr>;
4626 def : Pat<(VTI.Vec (int_arm_mve_vcmulq
4627 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
4628 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4631 def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated
4632 imm:$rot, (VTI.Vec MQPR:$inactive),
4633 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4634 (VTI.Pred VCCR:$mask))),
4635 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4636 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
4637 (VTI.Vec MQPR:$inactive)))>;
4666 multiclass MVE_VMULL_m<MVEVectorVTInfo VTI,
4669 def "" : MVE_VMULL<"vmull" # !if(Top, "t", "b"), VTI.Suffix, VTI.Unsigned,
4670 VTI.Size, Top, cstr, vecsize>;
4674 defvar uflag = !if(!eq(VTI.SuffixLetter, "p"), (?), (? (i32 VTI.Unsigned)));
4677 def : Pat<(VTI.DblVec !con((unpred_op (VTI.Vec MQPR:$Qm),
4678 (VTI.Vec MQPR:$Qn)),
4680 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4683 def : Pat<(VTI.DblVec !con((pred_int (VTI.Vec MQPR:$Qm),
4684 (VTI.Vec MQPR:$Qn)),
4685 uflag, (? (i32 Top), (VTI.DblPred VCCR:$mask),
4686 (VTI.DblVec MQPR:$inactive)))),
4687 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4688 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
4689 (VTI.DblVec MQPR:$inactive)))>;
4797 multiclass MVE_VxMULH_m<string iname, MVEVectorVTInfo VTI, SDNode unpred_op,
4799 def "" : MVE_VxMULH<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, round>;
4804 defvar mulh = !if(VTI.Unsigned, mulhu, mulhs);
4805 defm : MVE_TwoOpPattern<VTI, mulh, PredInt, (? (i32 VTI.Unsigned)),
4809 def : Pat<(VTI.Vec (PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4810 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
4811 (VTI.Vec MQPR:$inactive))),
4812 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4813 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
4814 (VTI.Vec MQPR:$inactive)))>;
4818 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4819 (i32 VTI.Unsigned))),
4820 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4824 multiclass MVE_VMULT<string iname, MVEVectorVTInfo VTI, bit round>
4825 : MVE_VxMULH_m<iname, VTI, !if(round, int_arm_mve_vrmulh, int_arm_mve_vmulh),
4881 MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4884 def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qd_src),
4885 (VTI.Vec MQPR:$Qm), (i32 top))),
4886 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4895 def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qm),
4896 (VTI.Vec (vrev MQPR:$Qd_src)), (i32 1))),
4897 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4903 def : Pat<(VTI.Vec (int_arm_mve_vmovn_predicated (VTI.Vec MQPR:$Qd_src),
4906 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4917 MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4918 def : Pat<(VTI.Vec (int_arm_mve_vqmovn (VTI.Vec MQPR:$Qd_src),
4921 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4924 def : Pat<(VTI.Vec (int_arm_mve_vqmovn_predicated (VTI.Vec MQPR:$Qd_src),
4928 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
5070 multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,
5072 def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;
5076 def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,
5077 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
5078 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5081 def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,
5082 imm:$rot, (VTI.Vec MQPR:$inactive),
5083 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5084 (VTI.Pred VCCR:$mask))),
5085 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5086 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
5087 (VTI.Vec MQPR:$inactive)))>;
5146 multiclass MVE_VQDMULL_m<string iname, MVEVectorVTInfo VTI, bit size, bit T,
5148 def "" : MVE_VQDMULL<iname, VTI.Suffix, size, T, cstr>;
5153 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5154 (VTI.Vec MQPR:$Qn), (i32 T))),
5155 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
5157 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5158 (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
5159 (i32 T), (VTI.DblPred VCCR:$mask),
5160 (VTI.DblVec MQPR:$inactive))),
5161 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
5162 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
5163 (VTI.DblVec MQPR:$inactive)))>;
5167 multiclass MVE_VQDMULL_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5168 defm bh : MVE_VQDMULL_m<"vqdmullb", VTI, size, 0b0, cstr>;
5169 defm th : MVE_VQDMULL_m<"vqdmullt", VTI, size, 0b1, cstr>;
5219 multiclass MVE_vec_scalar_int_pat_m<Instruction inst, MVEVectorVTInfo VTI,
5224 defvar UnpredSign = !if(unpred_has_sign, (? (i32 VTI.Unsigned)), (?));
5225 defvar PredSign = !if(pred_has_sign, (? (i32 VTI.Unsigned)), (?));
5229 def : Pat<(VTI.Vec !con((unpred_op (VTI.Vec MQPR:$Qm),
5230 (VTI.Vec (ARMvdup rGPR:$val))),
5232 (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5234 def : Pat<(VTI.Vec !con((pred_op (VTI.Vec MQPR:$Qm),
5235 (VTI.Vec (ARMvdup rGPR:$val))),
5237 (pred_op (VTI.Pred VCCR:$mask),
5238 (VTI.Vec MQPR:$inactive)))),
5239 (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5240 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
5241 (VTI.Vec MQPR:$inactive)))>;
5259 multiclass MVE_VADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5261 def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b0, subtract, 0b1, 0b0>;
5263 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
5267 multiclass MVE_VADD_qr_m<MVEVectorVTInfo VTI>
5268 : MVE_VADDSUB_qr_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
5270 multiclass MVE_VSUB_qr_m<MVEVectorVTInfo VTI>
5271 : MVE_VADDSUB_qr_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
5282 multiclass MVE_VQADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5284 def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b1, subtract,
5285 0b0, VTI.Unsigned>;
5288 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
5293 multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI, SDNode Op>
5294 : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, Op, int_arm_mve_qadd_predicated>;
5296 multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI, SDNode Op>
5297 : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, Op, int_arm_mve_qsub_predicated>;
5327 multiclass MVE_VQDMULL_qr_m<string iname, MVEVectorVTInfo VTI, bit size,
5329 def "" : MVE_VQDMULL_qr<iname, VTI.Suffix, size, T, cstr>;
5334 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5335 (VTI.Vec (ARMvdup rGPR:$val)),
5337 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5339 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5340 (VTI.Vec MQPR:$Qm),
5341 (VTI.Vec (ARMvdup rGPR:$val)),
5343 (VTI.DblPred VCCR:$mask),
5344 (VTI.DblVec MQPR:$inactive))),
5345 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5346 ARMVCCThen, (VTI.DblPred VCCR:$mask), zero_reg,
5347 (VTI.DblVec MQPR:$inactive)))>;
5351 multiclass MVE_VQDMULL_qr_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5352 defm bh : MVE_VQDMULL_qr_m<"vqdmullb", VTI, size, 0b0, cstr>;
5353 defm th : MVE_VQDMULL_qr_m<"vqdmullt", VTI, size, 0b1, cstr>;
5373 multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract, SDNode Op,
5375 def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract, VTI.Size>;
5376 defm : MVE_TwoOpPatternDup<VTI, Op, pred_int, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
5378 VTI, unpred_int, pred_int, 1, 1>;
5382 def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))), (i32 1))),
5387 multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op, SDNode Op> :
5388 MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, Op, int_arm_mve_vhadd,
5391 multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op> :
5392 MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, null_frag, int_arm_mve_vhsub,
5409 multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract,
5411 def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, subtract, VTI.Size>;
5412 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ),
5444 multiclass MVE_VxSHL_qr_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
5445 def "" : MVE_VxSHL_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
5448 def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar
5449 (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5450 (i32 q), (i32 r), (i32 VTI.Unsigned))),
5451 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh)))>;
5453 def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar_predicated
5454 (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5455 (i32 q), (i32 r), (i32 VTI.Unsigned),
5456 (VTI.Pred VCCR:$mask))),
5457 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5458 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg))>;
5507 multiclass MVE_VBRSR_pat_m<MVEVectorVTInfo VTI, Instruction Inst> {
5509 def : Pat<(VTI.Vec (int_arm_mve_vbrsr (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm))),
5510 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm)))>;
5512 def : Pat<(VTI.Vec (int_arm_mve_vbrsr_predicated
5513 (VTI.Vec MQPR:$inactive),
5514 (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5515 (VTI.Pred VCCR:$mask))),
5516 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5517 ARMVCCThen, (VTI.Pred VCCR:$mask), zero_reg,
5518 (VTI.Vec MQPR:$inactive)))>;
5553 multiclass MVE_VMUL_qr_int_m<MVEVectorVTInfo VTI> {
5554 def "" : MVE_VMUL_qr_int<"vmul", VTI.Suffix, VTI.Size>;
5556 defm : MVE_TwoOpPatternDup<VTI, mul, int_arm_mve_mul_predicated, (? ),
5578 multiclass MVE_VxxMUL_qr_m<string iname, MVEVectorVTInfo VTI, bit bit_28,
5580 def "" : MVE_VxxMUL_qr<iname, VTI.Suffix, bit_28, VTI.Size, VTI.Size>;
5583 defm : MVE_TwoOpPatternDup<VTI, Op, int_pred, (? ), !cast<Instruction>(NAME)>;
5585 defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI, int_unpred, int_pred>;
5588 multiclass MVE_VQDMULH_qr_m<MVEVectorVTInfo VTI> :
5589 MVE_VxxMUL_qr_m<"vqdmulh", VTI, 0b0, MVEvqdmulh,
5592 multiclass MVE_VQRDMULH_qr_m<MVEVectorVTInfo VTI> :
5593 MVE_VxxMUL_qr_m<"vqrdmulh", VTI, 0b1, null_frag,
5604 multiclass MVE_VxxMUL_qr_f_m<MVEVectorVTInfo VTI, SDPatternOperator IdentityVec> {
5606 def "" : MVE_VxxMUL_qr<"vmul", VTI.Suffix, VTI.Size{0}, 0b11, VTI.Size>;
5607 defm : MVE_TwoOpPatternDup<VTI, fmul, int_arm_mve_mul_predicated, (? ),
5631 multiclass MVE_VMLA_qr_multi<string iname, MVEVectorVTInfo VTI,
5633 def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, 0b0, VTI.Size,
5634 scalar_addend, VTI.Size>;
5637 defvar v1 = (VTI.Vec MQPR:$v1);
5638 defvar v2 = (VTI.Vec MQPR:$v2);
5639 defvar vs = (VTI.Vec (ARMvdup rGPR:$s));
5641 defvar pred = (VTI.Pred VCCR:$pred);
5645 def : Pat<(VTI.Vec (add (mul v1, v2), vs)),
5646 (VTI.Vec (Inst v1, v2, s))>;
5648 def : Pat<(VTI.Vec (add (mul v2, vs), v1)),
5649 (VTI.Vec (Inst v1, v2, s))>;
5652 def : Pat<(VTI.Vec (pred_int v1, v2, s, pred)),
5653 (VTI.Vec (Inst v1, v2, s, ARMVCCThen, pred, zero_reg))>;
5665 multiclass MVE_VFMA_qr_multi<string iname, MVEVectorVTInfo VTI,
5667 def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, scalar_addend, VTI.Size>;
5670 defvar v1 = (VTI.Vec MQPR:$v1);
5671 defvar v2 = (VTI.Vec MQPR:$v2);
5672 defvar vs = (VTI.Vec (ARMvdup (i32 rGPR:$s)));
5674 defvar pred = (VTI.Pred VCCR:$pred);
5678 def : Pat<(VTI.Vec (fma v1, v2, vs)),
5679 (VTI.Vec (Inst v1, v2, is))>;
5680 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5681 (VTI.Vec (fma v1, v2, vs)),
5683 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;
5684 def : Pat<(VTI.Vec (pred_int v1, v2, vs, pred)),
5685 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, pred, zero_reg))>;
5687 def : Pat<(VTI.Vec (fma v1, vs, v2)),
5688 (VTI.Vec (Inst v2, v1, is))>;
5689 def : Pat<(VTI.Vec (fma vs, v1, v2)),
5690 (VTI.Vec (Inst v2, v1, is))>;
5691 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5692 (VTI.Vec (fma vs, v2, v1)),
5694 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;
5695 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5696 (VTI.Vec (fma v2, vs, v1)),
5698 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred, zero_reg))>;
5699 def : Pat<(VTI.Vec (pred_int v1, vs, v2, pred)),
5700 (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred, zero_reg))>;
5701 def : Pat<(VTI.Vec (pred_int vs, v1, v2, pred)),
5702 (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred, zero_reg))>;
5726 multiclass MVE_VQDMLAH_qr_multi<string iname, MVEVectorVTInfo VTI,
5728 def "": MVE_VQDMLAH_qr<iname, VTI.Suffix, 0b0, VTI.Size, bit_5, bit_12>;
5734 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5736 (VTI.Vec (Inst (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5738 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5739 (i32 rGPR:$s), (VTI.Pred VCCR:$pred))),
5740 (VTI.Vec (Inst (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5742 (VTI.Pred VCCR:$pred), zero_reg))>;
5847 multiclass MVE_VCTP<MVEVectorVTInfo VTI, Intrinsic intr> {
5848 def "": MVE_VCTPInst<VTI.BitsSuffix, VTI.Size>;
5853 (VTI.Pred (Inst rGPR:$Rn))>;
5854 def : Pat<(and (intr rGPR:$Rn), (VTI.Pred VCCR:$mask)),
5855 (VTI.Pred (Inst rGPR:$Rn, ARMVCCThen, VCCR:$mask, zero_reg))>;
6382 foreach VTI = VTIs in
6383 foreach UnsignedFlag = !if(!eq(VTI.Size, memsz.encoding),
6384 [0,1], [VTI.Unsigned]) in {
6385 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag)),
6386 (VTI.Vec (InstU GPR:$base, MQPR:$offsets))>;
6387 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag)),
6388 (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6389 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, 0, UnsignedFlag, (VTI.Pred VCCR:$pred))),
6390 (VTI.Vec (InstU GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;
6391 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.TypeBits, memsz.shift, UnsignedFlag, (VTI.Pred VCCR:$pred))),
6392 (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;
6400 foreach VTI = VTIs in {
6401 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned)),
6402 (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6403 def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, VTI.Unsigned, (VTI.Pred VCCR:$pred))),
6404 (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred, zero_reg))>;
6413 foreach VTI = VTIs in {
6414 def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0),
6416 def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift),
6418 def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0, (VTI.Pred VCCR:$pred)),
6420 def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift, (VTI.Pred VCCR:$pred)),
6429 foreach VTI = VTIs in {
6430 def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0),
6432 def : Pat<(int_arm_mve_vstr_scatter_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0, (VTI.Pred VCCR:$pred)),