1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVELogicalImm8Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>; 203def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>; 204def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>; 205def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 206 207def SVELogicalImm8NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; 208def SVELogicalImm16NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; 209def SVELogicalImm32NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; 210def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; 211 212def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>; 213 214def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 215def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 216def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 217def SVEArithUImm64Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>; 218def SVEArithSImmPat : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 219 220def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 221def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 222def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 223def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>; 224def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 225def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 226def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 227def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>; 228 229def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; 230 231class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 232 let Name = "SVEExactFPImmOperand" # Suffix; 233 let DiagnosticType = "Invalid" # Name; 234 let ParserMethod = "tryParseFPImm<false>"; 235 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 236 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 237} 238 239class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 240 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 241 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 242} 243 244def sve_fpimm_half_one 245 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 246 "AArch64ExactFPImm::one">; 247def sve_fpimm_half_two 248 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 249 "AArch64ExactFPImm::two">; 250def sve_fpimm_zero_one 251 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 252 "AArch64ExactFPImm::one">; 253 254def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 255 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 256}]> { 257 let ParserMatchClass = Imm1_16Operand; 258 let EncoderMethod = "getSVEIncDecImm"; 259 let DecoderMethod = "DecodeSVEIncDecImm"; 260} 261 262// This allows i32 immediate extraction from i64 based arithmetic. 263def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 264def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">; 265 266 267def sve_ext_imm_0_31 : ComplexPattern<i32, 1, "SelectEXTImm<31, 8>">; 268def sve_ext_imm_0_63 : ComplexPattern<i32, 1, "SelectEXTImm<63, 4>">; 269def sve_ext_imm_0_127 : ComplexPattern<i32, 1, "SelectEXTImm<127, 2>">; 270def sve_ext_imm_0_255 : ComplexPattern<i32, 1, "SelectEXTImm<255, 1>">; 271 272def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), 273 (int_aarch64_sve_cntp node:$pred, node:$src2), [{ 274 return N->hasOneUse(); 275}]>; 276 277def step_vector_oneuse : PatFrag<(ops node:$idx), 278 (step_vector node:$idx), [{ 279 return N->hasOneUse(); 280}]>; 281 282 283//===----------------------------------------------------------------------===// 284// SVE PTrue - These are used extensively throughout the pattern matching so 285// it's important we define them first. 286//===----------------------------------------------------------------------===// 287 288class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 289 ValueType vt, SDPatternOperator op> 290: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 291 asm, "\t$Pd, $pattern", 292 "", 293 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 294 bits<4> Pd; 295 bits<5> pattern; 296 let Inst{31-24} = 0b00100101; 297 let Inst{23-22} = sz8_64; 298 let Inst{21-19} = 0b011; 299 let Inst{18-17} = opc{2-1}; 300 let Inst{16} = opc{0}; 301 let Inst{15-10} = 0b111000; 302 let Inst{9-5} = pattern; 303 let Inst{4} = 0b0; 304 let Inst{3-0} = Pd; 305 306 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 307 let ElementSize = pprty.ElementSize; 308 let isReMaterializable = 1; 309} 310 311multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 312 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 313 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 314 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 315 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 316 317 def : InstAlias<asm # "\t$Pd", 318 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 319 def : InstAlias<asm # "\t$Pd", 320 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 321 def : InstAlias<asm # "\t$Pd", 322 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 323 def : InstAlias<asm # "\t$Pd", 324 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 325} 326 327def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 328def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 329 330let Predicates = [HasSVEorStreamingSVE] in { 331 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 332 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 333} 334 335//===----------------------------------------------------------------------===// 336// SVE pattern match helpers. 337//===----------------------------------------------------------------------===// 338 339class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 340 Instruction inst> 341: Pat<(vtd (op vt1:$Op1)), 342 (inst $Op1)>; 343 344class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 345 ValueType vts, Instruction inst> 346: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 347 (inst $Op3, $Op1, $Op2)>; 348 349 350multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 351 ValueType vts, Instruction inst> { 352 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))), 353 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 354 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)), 355 (inst $Op3, $Op1, $Op2)>; 356} 357 358// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 359// type of rounding. This is matched by timm0_1 in pattern below and ignored. 360class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 361 ValueType vts, Instruction inst> 362: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 363 (inst $Op3, $Op1, $Op2)>; 364 365class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 366 ValueType it, ComplexPattern cpx, Instruction inst> 367 : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))), 368 (inst $Op1, i32:$imm, i32:$shift)>; 369 370class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 371 ValueType it, ComplexPattern cpx, Instruction inst> 372 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))), 373 (inst $Op1, i32:$imm, i32:$shift)>; 374 375class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op, 376 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 377 : Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))), 378 (inst $Op1, i32:$imm)>; 379 380class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 381 ValueType it, ComplexPattern cpx, Instruction inst> 382 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))), 383 (inst $Op1, i64:$imm)>; 384 385class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 386 ValueType vt2, Instruction inst> 387: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 388 (inst $Op1, $Op2)>; 389 390class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 391 ValueType pt, ValueType vt1, ValueType vt2, 392 Instruction inst> 393: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)), 394 (inst $Op1, $Op2)>; 395 396class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op, 397 ValueType pt, ValueType vt1, ValueType vt2, 398 Instruction inst> 399: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)), 400 (inst $Op1, $Op2, $Op3)>; 401 402class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 403 ValueType vt2, ValueType vt3, Instruction inst> 404: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 405 (inst $Op1, $Op2, $Op3)>; 406 407multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 408 ValueType vt2, ValueType vt3, Instruction inst> { 409 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)), 410 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 411 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)), 412 (inst $Op1, $Op2, $Op3)>; 413} 414 415class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 416 ValueType vt2, ValueType vt3, ValueType vt4, 417 Instruction inst> 418: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 419 (inst $Op1, $Op2, $Op3, $Op4)>; 420 421class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 422 ValueType vt2, Operand ImmTy, Instruction inst> 423: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 424 (inst $Op1, ImmTy:$Op2)>; 425 426class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 427 ValueType vt2, ValueType vt3, Operand ImmTy, 428 Instruction inst> 429: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 430 (inst $Op1, $Op2, ImmTy:$Op3)>; 431 432class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 433 ValueType vt2, ValueType vt3, ValueType vt4, 434 Operand ImmTy, Instruction inst> 435: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 436 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 437 438def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>; 439def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>; 440 441let AddedComplexity = 1 in { 442class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 443 ValueType vt2, ValueType vt3, Instruction inst> 444: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 445 (inst $Op1, $Op2, $Op3)>; 446 447class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 448 ValueType vt1, ValueType vt2, 449 Operand vt3, Instruction inst> 450: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 451 (inst $Op1, $Op2, vt3:$Op3)>; 452} 453 454// 455// Common but less generic patterns. 456// 457 458class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 459 Instruction inst, Instruction ptrue> 460: Pat<(vtd (op vt1:$Op1)), 461 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 462 463class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 464 ValueType vt2, Instruction inst, Instruction ptrue> 465: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 466 (inst (ptrue 31), $Op1, $Op2)>; 467 468class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 469 ValueType inreg_vt, Instruction inst> 470: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 471 (inst $PassThru, $Pg, $Src)>; 472 473multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt, 474 ValueType inreg_vt, Instruction inst> { 475 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))), 476 (inst (IMPLICIT_DEF), $Pg, $Src)>; 477 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)), 478 (inst $PassThru, $Pg, $Src)>; 479} 480 481class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 482 ValueType pt, ValueType it, 483 ComplexPattern cast, Instruction inst> 484: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))), 485 (inst $Pg, $Rn, i32:$imm)>; 486 487class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 488 ValueType pt, ValueType it, 489 ComplexPattern cast, Instruction inst> 490: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))), 491 (inst $Rn, i32:$imm)>; 492 493class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op, 494 ValueType pt, ValueType it, 495 FPImmLeaf immL, int imm, 496 Instruction inst> 497: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (AArch64dup (it immL))))), 498 (inst $Pg, $Zs1, imm)>; 499 500class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, 501 ValueType pt, ValueType it, 502 FPImmLeaf immL, int imm, 503 Instruction inst> 504: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)), 505 (vt (AArch64dup (it immL))))), 506 (inst $Pg, $Zs1, imm)>; 507 508// 509// Pseudo -> Instruction mappings 510// 511def getSVEPseudoMap : InstrMapping { 512 let FilterClass = "SVEPseudo2Instr"; 513 let RowFields = ["PseudoName"]; 514 let ColFields = ["IsInstr"]; 515 let KeyCol = ["0"]; 516 let ValueCols = [["1"]]; 517} 518 519class SVEPseudo2Instr<string name, bit instr> { 520 string PseudoName = name; 521 bit IsInstr = instr; 522} 523 524// Lookup e.g. DIV -> DIVR 525def getSVERevInstr : InstrMapping { 526 let FilterClass = "SVEInstr2Rev"; 527 let RowFields = ["InstrName"]; 528 let ColFields = ["isReverseInstr"]; 529 let KeyCol = ["0"]; 530 let ValueCols = [["1"]]; 531} 532 533// Lookup e.g. DIVR -> DIV 534def getSVENonRevInstr : InstrMapping { 535 let FilterClass = "SVEInstr2Rev"; 536 let RowFields = ["InstrName"]; 537 let ColFields = ["isReverseInstr"]; 538 let KeyCol = ["1"]; 539 let ValueCols = [["0"]]; 540} 541 542class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 543 string InstrName = !if(name1IsReverseInstr, name1, name2); 544 bit isReverseInstr = name1IsReverseInstr; 545} 546 547// 548// Pseudos for destructive operands 549// 550let hasNoSchedulingInfo = 1 in { 551 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 552 FalseLanesEnum flags = FalseLanesNone> 553 : SVEPseudo2Instr<name, 0>, 554 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 555 let FalseLanes = flags; 556 } 557 558 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 559 FalseLanesEnum flags = FalseLanesNone> 560 : SVEPseudo2Instr<name, 0>, 561 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 562 let FalseLanes = flags; 563 } 564 565 class PredThreeOpPseudo<string name, ZPRRegOp zprty, 566 FalseLanesEnum flags = FalseLanesNone> 567 : SVEPseudo2Instr<name, 0>, 568 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> { 569 let FalseLanes = flags; 570 } 571} 572 573// 574// Pseudos for passthru operands 575// 576let hasNoSchedulingInfo = 1 in { 577 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty> 578 : SVEPseudo2Instr<name, 0>, 579 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>; 580} 581 582//===----------------------------------------------------------------------===// 583// SVE Predicate Misc Group 584//===----------------------------------------------------------------------===// 585 586class sve_int_pfalse<bits<6> opc, string asm> 587: I<(outs PPR8:$Pd), (ins), 588 asm, "\t$Pd", 589 "", 590 []>, Sched<[]> { 591 bits<4> Pd; 592 let Inst{31-24} = 0b00100101; 593 let Inst{23-22} = opc{5-4}; 594 let Inst{21-19} = 0b011; 595 let Inst{18-16} = opc{3-1}; 596 let Inst{15-10} = 0b111001; 597 let Inst{9} = opc{0}; 598 let Inst{8-4} = 0b00000; 599 let Inst{3-0} = Pd; 600 601 let isReMaterializable = 1; 602} 603 604class sve_int_ptest<bits<6> opc, string asm> 605: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 606 asm, "\t$Pg, $Pn", 607 "", 608 []>, Sched<[]> { 609 bits<4> Pg; 610 bits<4> Pn; 611 let Inst{31-24} = 0b00100101; 612 let Inst{23-22} = opc{5-4}; 613 let Inst{21-19} = 0b010; 614 let Inst{18-16} = opc{3-1}; 615 let Inst{15-14} = 0b11; 616 let Inst{13-10} = Pg; 617 let Inst{9} = opc{0}; 618 let Inst{8-5} = Pn; 619 let Inst{4-0} = 0b00000; 620 621 let Defs = [NZCV]; 622 let isCompare = 1; 623} 624 625class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 626 PPRRegOp pprty> 627: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 628 asm, "\t$Pdn, $Pg, $_Pdn", 629 "", 630 []>, Sched<[]> { 631 bits<4> Pdn; 632 bits<4> Pg; 633 let Inst{31-24} = 0b00100101; 634 let Inst{23-22} = sz8_64; 635 let Inst{21-19} = 0b011; 636 let Inst{18-16} = opc{4-2}; 637 let Inst{15-11} = 0b11000; 638 let Inst{10-9} = opc{1-0}; 639 let Inst{8-5} = Pg; 640 let Inst{4} = 0; 641 let Inst{3-0} = Pdn; 642 643 let Constraints = "$Pdn = $_Pdn"; 644 let Defs = [NZCV]; 645 let isPTestLike = 1; 646 let ElementSize = pprty.ElementSize; 647} 648 649multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 650 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 651 652 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 653} 654 655multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 656 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 657 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 658 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 659 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 660 661 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 662 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 663 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 664 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 665} 666 667//===----------------------------------------------------------------------===// 668// SVE Predicate Count Group 669//===----------------------------------------------------------------------===// 670 671class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 672 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 673: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 674 asm, "\t$Rdn, $Pg", 675 "", 676 []>, Sched<[]> { 677 bits<5> Rdn; 678 bits<4> Pg; 679 let Inst{31-24} = 0b00100101; 680 let Inst{23-22} = sz8_64; 681 let Inst{21-19} = 0b101; 682 let Inst{18-16} = opc{4-2}; 683 let Inst{15-11} = 0b10001; 684 let Inst{10-9} = opc{1-0}; 685 let Inst{8-5} = Pg; 686 let Inst{4-0} = Rdn; 687 688 // Signed 32bit forms require their GPR operand printed. 689 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 690 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 691 !strconcat(asm, "\t$Rdn, $Pg")); 692 let Constraints = "$Rdn = $_Rdn"; 693} 694 695multiclass sve_int_count_r_s32<bits<5> opc, string asm, 696 SDPatternOperator op> { 697 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 698 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 699 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 700 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 701 702 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 703 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 704 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 705 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 706 707 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 708 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 709 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 710 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 711 712 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 713 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 714 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 715 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 716 717 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 718 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 719 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 720 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 721} 722 723multiclass sve_int_count_r_u32<bits<5> opc, string asm, 724 SDPatternOperator op> { 725 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 726 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 727 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 728 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 729 730 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 731 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 732 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 733 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 734 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 735 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 736 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 737 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 738} 739 740multiclass sve_int_count_r_x64<bits<5> opc, string asm, 741 SDPatternOperator op, 742 SDPatternOperator combine_op = null_frag> { 743 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 744 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 745 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 746 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 747 748 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 749 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 750 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 751 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 752 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 753 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 754 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 755 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 756 757 // Combine cntp with combine_op 758 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))), 759 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 760 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))), 761 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 762 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))), 763 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 764 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))), 765 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 766} 767 768class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 769 ZPRRegOp zprty, PPRRegOp pprty> 770: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 771 asm, "\t$Zdn, $Pm", 772 "", 773 []>, Sched<[]> { 774 bits<4> Pm; 775 bits<5> Zdn; 776 let Inst{31-24} = 0b00100101; 777 let Inst{23-22} = sz8_64; 778 let Inst{21-19} = 0b101; 779 let Inst{18-16} = opc{4-2}; 780 let Inst{15-11} = 0b10000; 781 let Inst{10-9} = opc{1-0}; 782 let Inst{8-5} = Pm; 783 let Inst{4-0} = Zdn; 784 785 let Constraints = "$Zdn = $_Zdn"; 786 let DestructiveInstType = DestructiveOther; 787 let ElementSize = ElementSizeNone; 788} 789 790multiclass sve_int_count_v<bits<5> opc, string asm, 791 SDPatternOperator op = null_frag> { 792 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 793 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 794 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 795 796 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 797 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 798 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 799 800 def : InstAlias<asm # "\t$Zdn, $Pm", 801 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 802 def : InstAlias<asm # "\t$Zdn, $Pm", 803 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 804 def : InstAlias<asm # "\t$Zdn, $Pm", 805 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 806} 807 808class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 809 PPRRegOp pprty> 810: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 811 asm, "\t$Rd, $Pg, $Pn", 812 "", 813 []>, Sched<[]> { 814 bits<4> Pg; 815 bits<4> Pn; 816 bits<5> Rd; 817 let Inst{31-24} = 0b00100101; 818 let Inst{23-22} = sz8_64; 819 let Inst{21-19} = 0b100; 820 let Inst{18-16} = opc{3-1}; 821 let Inst{15-14} = 0b10; 822 let Inst{13-10} = Pg; 823 let Inst{9} = opc{0}; 824 let Inst{8-5} = Pn; 825 let Inst{4-0} = Rd; 826} 827 828multiclass sve_int_pcount_pred<bits<4> opc, string asm, 829 SDPatternOperator int_op> { 830 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 831 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 832 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 833 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 834 835 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 836 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 837 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 838 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 839} 840 841//===----------------------------------------------------------------------===// 842// SVE Element Count Group 843//===----------------------------------------------------------------------===// 844 845class sve_int_count<bits<3> opc, string asm> 846: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 847 asm, "\t$Rd, $pattern, mul $imm4", 848 "", 849 []>, Sched<[]> { 850 bits<5> Rd; 851 bits<4> imm4; 852 bits<5> pattern; 853 let Inst{31-24} = 0b00000100; 854 let Inst{23-22} = opc{2-1}; 855 let Inst{21-20} = 0b10; 856 let Inst{19-16} = imm4; 857 let Inst{15-11} = 0b11100; 858 let Inst{10} = opc{0}; 859 let Inst{9-5} = pattern; 860 let Inst{4-0} = Rd; 861} 862 863multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 864 def NAME : sve_int_count<opc, asm>; 865 866 def : InstAlias<asm # "\t$Rd, $pattern", 867 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 868 def : InstAlias<asm # "\t$Rd", 869 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 870 871 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))), 872 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 873 874 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))), 875 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 876 877 def : Pat<(i64 (op sve_pred_enum:$pattern)), 878 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 879} 880 881class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 882: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 883 asm, "\t$Zdn, $pattern, mul $imm4", 884 "", 885 []>, Sched<[]> { 886 bits<5> Zdn; 887 bits<5> pattern; 888 bits<4> imm4; 889 let Inst{31-24} = 0b00000100; 890 let Inst{23-22} = opc{4-3}; 891 let Inst{21} = 0b1; 892 let Inst{20} = opc{2}; 893 let Inst{19-16} = imm4; 894 let Inst{15-12} = 0b1100; 895 let Inst{11-10} = opc{1-0}; 896 let Inst{9-5} = pattern; 897 let Inst{4-0} = Zdn; 898 899 let Constraints = "$Zdn = $_Zdn"; 900 let DestructiveInstType = DestructiveOther; 901 let ElementSize = ElementSizeNone; 902} 903 904multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 905 SDPatternOperator op = null_frag, 906 ValueType vt = OtherVT> { 907 def NAME : sve_int_countvlv<opc, asm, zprty>; 908 909 def : InstAlias<asm # "\t$Zdn, $pattern", 910 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 911 def : InstAlias<asm # "\t$Zdn", 912 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 913 914 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 915 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 916} 917 918class sve_int_pred_pattern_a<bits<3> opc, string asm> 919: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 920 asm, "\t$Rdn, $pattern, mul $imm4", 921 "", 922 []>, Sched<[]> { 923 bits<5> Rdn; 924 bits<5> pattern; 925 bits<4> imm4; 926 let Inst{31-24} = 0b00000100; 927 let Inst{23-22} = opc{2-1}; 928 let Inst{21-20} = 0b11; 929 let Inst{19-16} = imm4; 930 let Inst{15-11} = 0b11100; 931 let Inst{10} = opc{0}; 932 let Inst{9-5} = pattern; 933 let Inst{4-0} = Rdn; 934 935 let Constraints = "$Rdn = $_Rdn"; 936} 937 938multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, 939 SDPatternOperator op, 940 SDPatternOperator opcnt> { 941 let Predicates = [HasSVEorStreamingSVE] in { 942 def NAME : sve_int_pred_pattern_a<opc, asm>; 943 944 def : InstAlias<asm # "\t$Rdn, $pattern", 945 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 946 def : InstAlias<asm # "\t$Rdn", 947 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 948 } 949 950 let Predicates = [HasSVEorStreamingSVE, UseScalarIncVL] in { 951 def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), 952 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; 953 954 def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm)))), 955 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 956 957 def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm))))), 958 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 959 960 def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))), 961 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 962 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), 963 sub_32))>; 964 965 def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm i32:$imm)))), 966 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 967 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 968 sub_32))>; 969 970 def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (i64 (sve_cnt_shl_imm i32:$imm))))), 971 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 972 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 973 sub_32))>; 974 } 975} 976 977class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 978 RegisterOperand st> 979: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 980 asm, "\t$Rdn, $pattern, mul $imm4", 981 "", 982 []>, Sched<[]> { 983 bits<5> Rdn; 984 bits<5> pattern; 985 bits<4> imm4; 986 let Inst{31-24} = 0b00000100; 987 let Inst{23-22} = opc{4-3}; 988 let Inst{21} = 0b1; 989 let Inst{20} = opc{2}; 990 let Inst{19-16} = imm4; 991 let Inst{15-12} = 0b1111; 992 let Inst{11-10} = opc{1-0}; 993 let Inst{9-5} = pattern; 994 let Inst{4-0} = Rdn; 995 996 // Signed 32bit forms require their GPR operand printed. 997 let AsmString = !if(!eq(opc{2,0}, 0b00), 998 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 999 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 1000 1001 let Constraints = "$Rdn = $_Rdn"; 1002} 1003 1004multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 1005 SDPatternOperator op> { 1006 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 1007 1008 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 1009 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 1010 def : InstAlias<asm # "\t$Rd, $Rn", 1011 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 1012 1013 // NOTE: Register allocation doesn't like tied operands of differing register 1014 // class, hence the extra INSERT_SUBREG complication. 1015 1016 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1017 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 1018 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 1019 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1020} 1021 1022multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 1023 SDPatternOperator op> { 1024 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 1025 1026 def : InstAlias<asm # "\t$Rdn, $pattern", 1027 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1028 def : InstAlias<asm # "\t$Rdn", 1029 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 1030 1031 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1032 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1033} 1034 1035multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 1036 SDPatternOperator op> { 1037 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 1038 1039 def : InstAlias<asm # "\t$Rdn, $pattern", 1040 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1041 def : InstAlias<asm # "\t$Rdn", 1042 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 1043 1044 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1045 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1046} 1047 1048 1049//===----------------------------------------------------------------------===// 1050// SVE Permute - Cross Lane Group 1051//===----------------------------------------------------------------------===// 1052 1053class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1054 ValueType vt, RegisterClass srcRegType, 1055 SDPatternOperator op> 1056: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 1057 asm, "\t$Zd, $Rn", 1058 "", 1059 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 1060 bits<5> Rn; 1061 bits<5> Zd; 1062 let Inst{31-24} = 0b00000101; 1063 let Inst{23-22} = sz8_64; 1064 let Inst{21-10} = 0b100000001110; 1065 let Inst{9-5} = Rn; 1066 let Inst{4-0} = Zd; 1067} 1068 1069multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 1070 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 1071 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 1072 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 1073 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 1074 1075 def : InstAlias<"mov $Zd, $Rn", 1076 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 1077 def : InstAlias<"mov $Zd, $Rn", 1078 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 1079 def : InstAlias<"mov $Zd, $Rn", 1080 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 1081 def : InstAlias<"mov $Zd, $Rn", 1082 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 1083} 1084 1085class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 1086 ZPRRegOp zprty> 1087: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 1088 asm, "\t$Zd, $Zn$idx", 1089 "", 1090 []>, Sched<[]> { 1091 bits<5> Zd; 1092 bits<5> Zn; 1093 bits<7> idx; 1094 let Inst{31-24} = 0b00000101; 1095 let Inst{23-22} = {?,?}; // imm3h 1096 let Inst{21} = 0b1; 1097 let Inst{20-16} = tsz; 1098 let Inst{15-10} = 0b001000; 1099 let Inst{9-5} = Zn; 1100 let Inst{4-0} = Zd; 1101} 1102 1103multiclass sve_int_perm_dup_i<string asm> { 1104 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 1105 let Inst{23-22} = idx{5-4}; 1106 let Inst{20-17} = idx{3-0}; 1107 } 1108 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 1109 let Inst{23-22} = idx{4-3}; 1110 let Inst{20-18} = idx{2-0}; 1111 } 1112 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 1113 let Inst{23-22} = idx{3-2}; 1114 let Inst{20-19} = idx{1-0}; 1115 } 1116 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 1117 let Inst{23-22} = idx{2-1}; 1118 let Inst{20} = idx{0}; 1119 } 1120 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 1121 let Inst{23-22} = idx{1-0}; 1122 } 1123 1124 def : InstAlias<"mov $Zd, $Zn$idx", 1125 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 1126 def : InstAlias<"mov $Zd, $Zn$idx", 1127 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 1128 def : InstAlias<"mov $Zd, $Zn$idx", 1129 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1130 def : InstAlias<"mov $Zd, $Zn$idx", 1131 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1132 def : InstAlias<"mov $Zd, $Zn$idx", 1133 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1134 def : InstAlias<"mov $Zd, $Bn", 1135 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1136 def : InstAlias<"mov $Zd, $Hn", 1137 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1138 def : InstAlias<"mov $Zd, $Sn", 1139 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1140 def : InstAlias<"mov $Zd, $Dn", 1141 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1142 def : InstAlias<"mov $Zd, $Qn", 1143 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1144 1145 // Duplicate extracted element of vector into all vector elements 1146 def : Pat<(nxv16i8 (AArch64dup (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))), 1147 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>; 1148 def : Pat<(nxv8i16 (AArch64dup (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1149 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1150 def : Pat<(nxv4i32 (AArch64dup (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1151 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1152 def : Pat<(nxv2i64 (AArch64dup (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1153 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1154 def : Pat<(nxv8f16 (AArch64dup (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1155 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1156 def : Pat<(nxv8bf16 (AArch64dup (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1157 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1158 def : Pat<(nxv4f16 (AArch64dup (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1159 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1160 def : Pat<(nxv2f16 (AArch64dup (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1161 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1162 def : Pat<(nxv4f32 (AArch64dup (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1163 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1164 def : Pat<(nxv2f32 (AArch64dup (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1165 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1166 def : Pat<(nxv2f64 (AArch64dup (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1167 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1168} 1169 1170class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1171 RegisterOperand VecList> 1172: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1173 asm, "\t$Zd, $Zn, $Zm", 1174 "", 1175 []>, Sched<[]> { 1176 bits<5> Zd; 1177 bits<5> Zm; 1178 bits<5> Zn; 1179 let Inst{31-24} = 0b00000101; 1180 let Inst{23-22} = sz8_64; 1181 let Inst{21} = 0b1; 1182 let Inst{20-16} = Zm; 1183 let Inst{15-13} = 0b001; 1184 let Inst{12-11} = opc; 1185 let Inst{10} = 0b0; 1186 let Inst{9-5} = Zn; 1187 let Inst{4-0} = Zd; 1188} 1189 1190multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1191 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1192 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1193 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1194 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1195 1196 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1197 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1198 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1199 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1200 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1201 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1202 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1203 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1204 1205 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1206 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1207 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1208 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1209 1210 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1211 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1212 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1213 1214 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1215} 1216 1217multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1218 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1219 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1220 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1221 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1222 1223 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1224 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1225 nxv16i8:$Op2, zsub1), 1226 nxv16i8:$Op3))>; 1227 1228 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1229 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1230 nxv8i16:$Op2, zsub1), 1231 nxv8i16:$Op3))>; 1232 1233 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1234 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1235 nxv4i32:$Op2, zsub1), 1236 nxv4i32:$Op3))>; 1237 1238 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1239 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1240 nxv2i64:$Op2, zsub1), 1241 nxv2i64:$Op3))>; 1242 1243 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1244 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1245 nxv8f16:$Op2, zsub1), 1246 nxv8i16:$Op3))>; 1247 1248 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1249 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1250 nxv4f32:$Op2, zsub1), 1251 nxv4i32:$Op3))>; 1252 1253 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1254 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1255 nxv2f64:$Op2, zsub1), 1256 nxv2i64:$Op3))>; 1257 1258 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1259 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1260 nxv8bf16:$Op2, zsub1), 1261 nxv8i16:$Op3))>; 1262} 1263 1264class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1265: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1266 asm, "\t$Zd, $Zn, $Zm", 1267 "", 1268 []>, Sched<[]> { 1269 bits<5> Zd; 1270 bits<5> Zm; 1271 bits<5> Zn; 1272 let Inst{31-24} = 0b00000101; 1273 let Inst{23-22} = sz8_64; 1274 let Inst{21} = 0b1; 1275 let Inst{20-16} = Zm; 1276 let Inst{15-10} = 0b001011; 1277 let Inst{9-5} = Zn; 1278 let Inst{4-0} = Zd; 1279 1280 let Constraints = "$Zd = $_Zd"; 1281} 1282 1283multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1284 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1285 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1286 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1287 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1288 1289 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1290 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1291 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1292 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1293 1294 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1295 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1296 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1297 1298 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1299} 1300 1301class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1302: I<(outs zprty:$Zd), (ins zprty:$Zn), 1303 asm, "\t$Zd, $Zn", 1304 "", 1305 []>, Sched<[]> { 1306 bits<5> Zd; 1307 bits<5> Zn; 1308 let Inst{31-24} = 0b00000101; 1309 let Inst{23-22} = sz8_64; 1310 let Inst{21-10} = 0b111000001110; 1311 let Inst{9-5} = Zn; 1312 let Inst{4-0} = Zd; 1313} 1314 1315multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1316 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1317 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1318 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1319 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1320 1321 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1322 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1323 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1324 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1325 1326 def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>; 1327 def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>; 1328 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1329 def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>; 1330 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1331 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1332 1333 def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>; 1334 def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>; 1335 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1336} 1337 1338class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1339: I<(outs pprty:$Pd), (ins pprty:$Pn), 1340 asm, "\t$Pd, $Pn", 1341 "", 1342 []>, Sched<[]> { 1343 bits<4> Pd; 1344 bits<4> Pn; 1345 let Inst{31-24} = 0b00000101; 1346 let Inst{23-22} = sz8_64; 1347 let Inst{21-9} = 0b1101000100000; 1348 let Inst{8-5} = Pn; 1349 let Inst{4} = 0b0; 1350 let Inst{3-0} = Pd; 1351} 1352 1353multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1354 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1355 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1356 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1357 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1358 1359 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1360 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1361 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1362 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1363} 1364 1365class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1366 ZPRRegOp zprty1, ZPRRegOp zprty2> 1367: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1368 asm, "\t$Zd, $Zn", 1369 "", []>, Sched<[]> { 1370 bits<5> Zd; 1371 bits<5> Zn; 1372 let Inst{31-24} = 0b00000101; 1373 let Inst{23-22} = sz16_64; 1374 let Inst{21-18} = 0b1100; 1375 let Inst{17-16} = opc; 1376 let Inst{15-10} = 0b001110; 1377 let Inst{9-5} = Zn; 1378 let Inst{4-0} = Zd; 1379} 1380 1381multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1382 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1383 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1384 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1385 1386 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1387 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1388 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1389} 1390 1391class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1392 RegisterClass srcRegType> 1393: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1394 asm, "\t$Zdn, $Rm", 1395 "", 1396 []>, Sched<[]> { 1397 bits<5> Rm; 1398 bits<5> Zdn; 1399 let Inst{31-24} = 0b00000101; 1400 let Inst{23-22} = sz8_64; 1401 let Inst{21-10} = 0b100100001110; 1402 let Inst{9-5} = Rm; 1403 let Inst{4-0} = Zdn; 1404 1405 let Constraints = "$Zdn = $_Zdn"; 1406 let DestructiveInstType = DestructiveOther; 1407} 1408 1409multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1410 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1411 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1412 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1413 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1414 1415 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1416 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1417 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1418 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1419} 1420 1421class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1422 FPRasZPROperand srcOpType> 1423: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm), 1424 asm, "\t$Zdn, $Vm", 1425 "", 1426 []>, Sched<[]> { 1427 bits<5> Vm; 1428 bits<5> Zdn; 1429 let Inst{31-24} = 0b00000101; 1430 let Inst{23-22} = sz8_64; 1431 let Inst{21-10} = 0b110100001110; 1432 let Inst{9-5} = Vm; 1433 let Inst{4-0} = Zdn; 1434 1435 let Constraints = "$Zdn = $_Zdn"; 1436 let DestructiveInstType = DestructiveOther; 1437} 1438 1439multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1440 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>; 1441 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>; 1442 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>; 1443 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>; 1444 1445 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)), 1446 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1447 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)), 1448 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>; 1449 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)), 1450 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>; 1451 1452 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)), 1453 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1454 1455 // Keep integer insertions within the vector unit. 1456 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))), 1457 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>; 1458 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))), 1459 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>; 1460 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))), 1461 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>; 1462 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))), 1463 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>; 1464 1465} 1466 1467//===----------------------------------------------------------------------===// 1468// SVE Permute - Extract Group 1469//===----------------------------------------------------------------------===// 1470 1471class sve_int_perm_extract_i<string asm> 1472: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1473 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1474 "", []>, Sched<[]> { 1475 bits<5> Zdn; 1476 bits<5> Zm; 1477 bits<8> imm8; 1478 let Inst{31-21} = 0b00000101001; 1479 let Inst{20-16} = imm8{7-3}; 1480 let Inst{15-13} = 0b000; 1481 let Inst{12-10} = imm8{2-0}; 1482 let Inst{9-5} = Zm; 1483 let Inst{4-0} = Zdn; 1484 1485 let Constraints = "$Zdn = $_Zdn"; 1486 let DestructiveInstType = DestructiveOther; 1487 let ElementSize = ElementSizeNone; 1488} 1489 1490multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1491 def NAME : sve_int_perm_extract_i<asm>; 1492 1493 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1494 !cast<Instruction>(NAME)>; 1495} 1496 1497class sve2_int_perm_extract_i_cons<string asm> 1498: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1499 asm, "\t$Zd, $Zn, $imm8", 1500 "", []>, Sched<[]> { 1501 bits<5> Zd; 1502 bits<5> Zn; 1503 bits<8> imm8; 1504 let Inst{31-21} = 0b00000101011; 1505 let Inst{20-16} = imm8{7-3}; 1506 let Inst{15-13} = 0b000; 1507 let Inst{12-10} = imm8{2-0}; 1508 let Inst{9-5} = Zn; 1509 let Inst{4-0} = Zd; 1510} 1511 1512//===----------------------------------------------------------------------===// 1513// SVE Vector Select Group 1514//===----------------------------------------------------------------------===// 1515 1516class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1517: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1518 asm, "\t$Zd, $Pg, $Zn, $Zm", 1519 "", 1520 []>, Sched<[]> { 1521 bits<4> Pg; 1522 bits<5> Zd; 1523 bits<5> Zm; 1524 bits<5> Zn; 1525 let Inst{31-24} = 0b00000101; 1526 let Inst{23-22} = sz8_64; 1527 let Inst{21} = 0b1; 1528 let Inst{20-16} = Zm; 1529 let Inst{15-14} = 0b11; 1530 let Inst{13-10} = Pg; 1531 let Inst{9-5} = Zn; 1532 let Inst{4-0} = Zd; 1533} 1534 1535multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1536 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1537 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1538 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1539 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1540 1541 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1542 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1543 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1544 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1545 1546 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1547 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 1548 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1549 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 1550 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1551 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1552 1553 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1554 1555 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1556 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1557 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1558 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1559 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1560 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1561 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1562 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1563} 1564 1565 1566//===----------------------------------------------------------------------===// 1567// SVE Predicate Logical Operations Group 1568//===----------------------------------------------------------------------===// 1569 1570class sve_int_pred_log<bits<4> opc, string asm> 1571: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1572 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1573 "", 1574 []>, Sched<[]> { 1575 bits<4> Pd; 1576 bits<4> Pg; 1577 bits<4> Pm; 1578 bits<4> Pn; 1579 let Inst{31-24} = 0b00100101; 1580 let Inst{23-22} = opc{3-2}; 1581 let Inst{21-20} = 0b00; 1582 let Inst{19-16} = Pm; 1583 let Inst{15-14} = 0b01; 1584 let Inst{13-10} = Pg; 1585 let Inst{9} = opc{1}; 1586 let Inst{8-5} = Pn; 1587 let Inst{4} = opc{0}; 1588 let Inst{3-0} = Pd; 1589 1590 // SEL has no predication qualifier. 1591 let AsmString = !if(!eq(opc, 0b0011), 1592 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1593 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1594 1595 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1596 1597} 1598 1599multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1600 SDPatternOperator op_nopred = null_frag> { 1601 def NAME : sve_int_pred_log<opc, asm>; 1602 1603 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1604 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1605 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1606 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1607 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1608 !cast<Instruction>(NAME), PTRUE_B>; 1609 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1610 !cast<Instruction>(NAME), PTRUE_H>; 1611 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1612 !cast<Instruction>(NAME), PTRUE_S>; 1613 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1614 !cast<Instruction>(NAME), PTRUE_D>; 1615} 1616 1617//===----------------------------------------------------------------------===// 1618// SVE Logical Mask Immediate Group 1619//===----------------------------------------------------------------------===// 1620 1621class sve_int_log_imm<bits<2> opc, string asm> 1622: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1623 asm, "\t$Zdn, $_Zdn, $imms13", 1624 "", []>, Sched<[]> { 1625 bits<5> Zdn; 1626 bits<13> imms13; 1627 let Inst{31-24} = 0b00000101; 1628 let Inst{23-22} = opc; 1629 let Inst{21-18} = 0b0000; 1630 let Inst{17-5} = imms13; 1631 let Inst{4-0} = Zdn; 1632 1633 let Constraints = "$Zdn = $_Zdn"; 1634 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1635 let DestructiveInstType = DestructiveOther; 1636 let ElementSize = ElementSizeNone; 1637} 1638 1639multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1640 def NAME : sve_int_log_imm<opc, asm>; 1641 1642 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1643 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1644 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1645 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1646 1647 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1648 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1649 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1650 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1651 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1652 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1653 1654 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1655 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1656 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1657 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1658 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1659 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1660 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1661 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1662} 1663 1664multiclass sve_int_log_imm_bic<SDPatternOperator op> { 1665 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>; 1666 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>; 1667 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>; 1668 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>; 1669} 1670 1671class sve_int_dup_mask_imm<string asm> 1672: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1673 asm, "\t$Zd, $imms", 1674 "", 1675 []>, Sched<[]> { 1676 bits<5> Zd; 1677 bits<13> imms; 1678 let Inst{31-18} = 0b00000101110000; 1679 let Inst{17-5} = imms; 1680 let Inst{4-0} = Zd; 1681 1682 let isReMaterializable = 1; 1683 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1684} 1685 1686multiclass sve_int_dup_mask_imm<string asm> { 1687 def NAME : sve_int_dup_mask_imm<asm>; 1688 1689 def : InstAlias<"dupm $Zd, $imm", 1690 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1691 def : InstAlias<"dupm $Zd, $imm", 1692 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1693 def : InstAlias<"dupm $Zd, $imm", 1694 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1695 1696 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1697 def : InstAlias<"mov $Zd, $imm", 1698 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1699 def : InstAlias<"mov $Zd, $imm", 1700 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1701 def : InstAlias<"mov $Zd, $imm", 1702 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1703} 1704 1705//===----------------------------------------------------------------------===// 1706// SVE Integer Arithmetic - Unpredicated Group. 1707//===----------------------------------------------------------------------===// 1708 1709class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1710 ZPRRegOp zprty> 1711: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1712 asm, "\t$Zd, $Zn, $Zm", 1713 "", []>, Sched<[]> { 1714 bits<5> Zd; 1715 bits<5> Zm; 1716 bits<5> Zn; 1717 let Inst{31-24} = 0b00000100; 1718 let Inst{23-22} = sz8_64; 1719 let Inst{21} = 0b1; 1720 let Inst{20-16} = Zm; 1721 let Inst{15-13} = 0b000; 1722 let Inst{12-10} = opc; 1723 let Inst{9-5} = Zn; 1724 let Inst{4-0} = Zd; 1725} 1726 1727multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> { 1728 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1729 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1730 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1731 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1732 1733 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1734 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1735 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1736 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1737} 1738 1739//===----------------------------------------------------------------------===// 1740// SVE Floating Point Arithmetic - Predicated Group 1741//===----------------------------------------------------------------------===// 1742 1743class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1744 ZPRRegOp zprty, 1745 Operand imm_ty> 1746: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1747 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1748 "", 1749 []>, Sched<[]> { 1750 bits<3> Pg; 1751 bits<5> Zdn; 1752 bit i1; 1753 let Inst{31-24} = 0b01100101; 1754 let Inst{23-22} = sz; 1755 let Inst{21-19} = 0b011; 1756 let Inst{18-16} = opc; 1757 let Inst{15-13} = 0b100; 1758 let Inst{12-10} = Pg; 1759 let Inst{9-6} = 0b0000; 1760 let Inst{5} = i1; 1761 let Inst{4-0} = Zdn; 1762 1763 let Constraints = "$Zdn = $_Zdn"; 1764 let DestructiveInstType = DestructiveOther; 1765 let ElementSize = zprty.ElementSize; 1766} 1767 1768multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1769 let DestructiveInstType = DestructiveBinaryImm in { 1770 def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1771 def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1772 def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1773 } 1774 1775 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>; 1776 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>; 1777 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>; 1778 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>; 1779 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>; 1780 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>; 1781} 1782 1783class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1784 ZPRRegOp zprty> 1785: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1786 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1787 "", 1788 []>, Sched<[]> { 1789 bits<3> Pg; 1790 bits<5> Zdn; 1791 bits<5> Zm; 1792 let Inst{31-24} = 0b01100101; 1793 let Inst{23-22} = sz; 1794 let Inst{21-20} = 0b00; 1795 let Inst{19-16} = opc; 1796 let Inst{15-13} = 0b100; 1797 let Inst{12-10} = Pg; 1798 let Inst{9-5} = Zm; 1799 let Inst{4-0} = Zdn; 1800 1801 let Constraints = "$Zdn = $_Zdn"; 1802 let DestructiveInstType = DestructiveOther; 1803 let ElementSize = zprty.ElementSize; 1804} 1805 1806multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1807 SDPatternOperator op, DestructiveInstTypeEnum flags, 1808 string revname="", bit isReverseInstr=0> { 1809 let DestructiveInstType = flags in { 1810 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1811 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1812 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1813 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1814 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1815 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1816 } 1817 1818 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1819 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1820 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1821} 1822 1823multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1824 SDPatternOperator op> { 1825 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1826 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1827 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1828 1829 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1830 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1831 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1832} 1833 1834multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1835 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1836 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1837 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1838 1839 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1840 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1841 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1842} 1843 1844class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1845: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3), 1846 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1847 "", 1848 []>, Sched<[]> { 1849 bits<5> Zdn; 1850 bits<5> Zm; 1851 bits<3> imm3; 1852 let Inst{31-24} = 0b01100101; 1853 let Inst{23-22} = sz; 1854 let Inst{21-19} = 0b010; 1855 let Inst{18-16} = imm3; 1856 let Inst{15-10} = 0b100000; 1857 let Inst{9-5} = Zm; 1858 let Inst{4-0} = Zdn; 1859 1860 let Constraints = "$Zdn = $_Zdn"; 1861 let DestructiveInstType = DestructiveOther; 1862 let ElementSize = ElementSizeNone; 1863} 1864 1865multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1866 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1867 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1868 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1869 1870 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))), 1871 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>; 1872 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))), 1873 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>; 1874 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))), 1875 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>; 1876} 1877 1878multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> { 1879 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>; 1880 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>; 1881 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>; 1882 1883 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1884 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1885 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1886 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1887 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 1888 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 1889 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1890 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1891 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 1892 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 1893 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_UNDEF_D")>; 1894 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_UNDEF_D")>; 1895} 1896 1897multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1898 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>; 1899 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>; 1900 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>; 1901 1902 let AddedComplexity = 2 in { 1903 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_ZERO_H")>; 1904 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_ZERO_H")>; 1905 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_ZERO_S")>; 1906 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_ZERO_S")>; 1907 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_ZERO_D")>; 1908 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_ZERO_D")>; 1909 } 1910} 1911 1912//===----------------------------------------------------------------------===// 1913// SVE Floating Point Arithmetic - Unpredicated Group 1914//===----------------------------------------------------------------------===// 1915 1916class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 1917: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1918 asm, "\t$Zd, $Zn, $Zm", 1919 "", 1920 []>, Sched<[]> { 1921 bits<5> Zd; 1922 bits<5> Zm; 1923 bits<5> Zn; 1924 let Inst{31-24} = 0b01100101; 1925 let Inst{23-22} = sz; 1926 let Inst{21} = 0b0; 1927 let Inst{20-16} = Zm; 1928 let Inst{15-13} = 0b000; 1929 let Inst{12-10} = opc; 1930 let Inst{9-5} = Zn; 1931 let Inst{4-0} = Zd; 1932} 1933 1934multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 1935 SDPatternOperator predicated_op = null_frag> { 1936 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1937 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1938 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1939 1940 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1941 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1942 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1943 1944 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1945 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1946 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1947} 1948 1949multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 1950 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1951 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1952 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1953 1954 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1955 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1956 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1957} 1958 1959//===----------------------------------------------------------------------===// 1960// SVE Floating Point Fused Multiply-Add Group 1961//===----------------------------------------------------------------------===// 1962 1963class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 1964: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 1965 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 1966 "", 1967 []>, Sched<[]> { 1968 bits<3> Pg; 1969 bits<5> Zda; 1970 bits<5> Zm; 1971 bits<5> Zn; 1972 let Inst{31-24} = 0b01100101; 1973 let Inst{23-22} = sz; 1974 let Inst{21} = 0b1; 1975 let Inst{20-16} = Zm; 1976 let Inst{15} = 0b0; 1977 let Inst{14-13} = opc; 1978 let Inst{12-10} = Pg; 1979 let Inst{9-5} = Zn; 1980 let Inst{4-0} = Zda; 1981 1982 let Constraints = "$Zda = $_Zda"; 1983 let ElementSize = zprty.ElementSize; 1984} 1985 1986multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps, 1987 SDPatternOperator op, string revname, 1988 bit isReverseInstr=0> { 1989 let DestructiveInstType = DestructiveTernaryCommWithRev in { 1990 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>, 1991 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1992 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>, 1993 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1994 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>, 1995 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1996 } 1997 1998 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1999 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2000 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2001} 2002 2003class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 2004 ZPRRegOp zprty> 2005: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2006 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2007 "", 2008 []>, Sched<[]> { 2009 bits<3> Pg; 2010 bits<5> Za; 2011 bits<5> Zdn; 2012 bits<5> Zm; 2013 let Inst{31-24} = 0b01100101; 2014 let Inst{23-22} = sz; 2015 let Inst{21} = 0b1; 2016 let Inst{20-16} = Za; 2017 let Inst{15} = 0b1; 2018 let Inst{14-13} = opc; 2019 let Inst{12-10} = Pg; 2020 let Inst{9-5} = Zm; 2021 let Inst{4-0} = Zdn; 2022 2023 let Constraints = "$Zdn = $_Zdn"; 2024 let DestructiveInstType = DestructiveOther; 2025 let ElementSize = zprty.ElementSize; 2026} 2027 2028multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op, 2029 string revname, bit isReverseInstr> { 2030 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>, 2031 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2032 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>, 2033 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2034 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>, 2035 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2036 2037 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2038 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2039 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2040} 2041 2042multiclass sve_fp_3op_p_zds_zx { 2043 def _UNDEF_H : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 2044 def _UNDEF_S : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 2045 def _UNDEF_D : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 2046} 2047 2048//===----------------------------------------------------------------------===// 2049// SVE Floating Point Multiply-Add - Indexed Group 2050//===----------------------------------------------------------------------===// 2051 2052class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 2053 ZPRRegOp zprty1, 2054 ZPRRegOp zprty2, Operand itype> 2055: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 2056 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2057 bits<5> Zda; 2058 bits<5> Zn; 2059 let Inst{31-24} = 0b01100100; 2060 let Inst{23-22} = sz; 2061 let Inst{21} = 0b1; 2062 let Inst{15-11} = 0; 2063 let Inst{10} = opc; 2064 let Inst{9-5} = Zn; 2065 let Inst{4-0} = Zda; 2066 2067 let Constraints = "$Zda = $_Zda"; 2068 let DestructiveInstType = DestructiveOther; 2069 let ElementSize = ElementSizeNone; 2070} 2071 2072multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 2073 SDPatternOperator op> { 2074 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2075 bits<3> Zm; 2076 bits<3> iop; 2077 let Inst{22} = iop{2}; 2078 let Inst{20-19} = iop{1-0}; 2079 let Inst{18-16} = Zm; 2080 } 2081 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2082 bits<3> Zm; 2083 bits<2> iop; 2084 let Inst{20-19} = iop; 2085 let Inst{18-16} = Zm; 2086 } 2087 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2088 bits<4> Zm; 2089 bit iop; 2090 let Inst{20} = iop; 2091 let Inst{19-16} = Zm; 2092 } 2093 2094 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 2095 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 2096 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2097 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2098 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2099 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2100} 2101 2102 2103//===----------------------------------------------------------------------===// 2104// SVE Floating Point Multiply - Indexed Group 2105//===----------------------------------------------------------------------===// 2106 2107class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 2108 ZPRRegOp zprty2, Operand itype> 2109: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 2110 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2111 bits<5> Zd; 2112 bits<5> Zn; 2113 let Inst{31-24} = 0b01100100; 2114 let Inst{23-22} = sz; 2115 let Inst{21} = 0b1; 2116 let Inst{15-10} = 0b001000; 2117 let Inst{9-5} = Zn; 2118 let Inst{4-0} = Zd; 2119} 2120 2121multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 2122 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2123 bits<3> Zm; 2124 bits<3> iop; 2125 let Inst{22} = iop{2}; 2126 let Inst{20-19} = iop{1-0}; 2127 let Inst{18-16} = Zm; 2128 } 2129 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2130 bits<3> Zm; 2131 bits<2> iop; 2132 let Inst{20-19} = iop; 2133 let Inst{18-16} = Zm; 2134 } 2135 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2136 bits<4> Zm; 2137 bit iop; 2138 let Inst{20} = iop; 2139 let Inst{19-16} = Zm; 2140 } 2141 2142 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 2143 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 2144 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 2145 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 2146 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 2147 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 2148} 2149 2150//===----------------------------------------------------------------------===// 2151// SVE Floating Point Complex Multiply-Add Group 2152//===----------------------------------------------------------------------===// 2153 2154class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 2155: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 2156 complexrotateop:$imm), 2157 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 2158 "", []>, Sched<[]> { 2159 bits<5> Zda; 2160 bits<3> Pg; 2161 bits<5> Zn; 2162 bits<5> Zm; 2163 bits<2> imm; 2164 let Inst{31-24} = 0b01100100; 2165 let Inst{23-22} = sz; 2166 let Inst{21} = 0; 2167 let Inst{20-16} = Zm; 2168 let Inst{15} = 0; 2169 let Inst{14-13} = imm; 2170 let Inst{12-10} = Pg; 2171 let Inst{9-5} = Zn; 2172 let Inst{4-0} = Zda; 2173 2174 let Constraints = "$Zda = $_Zda"; 2175 let DestructiveInstType = DestructiveOther; 2176 let ElementSize = zprty.ElementSize; 2177} 2178 2179multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 2180 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 2181 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 2182 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 2183 2184 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 2185 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2186 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 2187 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2188 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 2189 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2190} 2191 2192//===----------------------------------------------------------------------===// 2193// SVE Floating Point Complex Multiply-Add - Indexed Group 2194//===----------------------------------------------------------------------===// 2195 2196class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 2197 ZPRRegOp zprty, 2198 ZPRRegOp zprty2, Operand itype> 2199: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 2200 complexrotateop:$imm), 2201 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 2202 "", []>, Sched<[]> { 2203 bits<5> Zda; 2204 bits<5> Zn; 2205 bits<2> imm; 2206 let Inst{31-24} = 0b01100100; 2207 let Inst{23-22} = sz; 2208 let Inst{21} = 0b1; 2209 let Inst{15-12} = 0b0001; 2210 let Inst{11-10} = imm; 2211 let Inst{9-5} = Zn; 2212 let Inst{4-0} = Zda; 2213 2214 let Constraints = "$Zda = $_Zda"; 2215 let DestructiveInstType = DestructiveOther; 2216 let ElementSize = ElementSizeNone; 2217} 2218 2219multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 2220 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 2221 bits<3> Zm; 2222 bits<2> iop; 2223 let Inst{20-19} = iop; 2224 let Inst{18-16} = Zm; 2225 } 2226 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 2227 bits<4> Zm; 2228 bits<1> iop; 2229 let Inst{20} = iop; 2230 let Inst{19-16} = Zm; 2231 } 2232 2233 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2234 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2235 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2236 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2237} 2238 2239//===----------------------------------------------------------------------===// 2240// SVE Floating Point Complex Addition Group 2241//===----------------------------------------------------------------------===// 2242 2243class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2244: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2245 complexrotateopodd:$imm), 2246 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2247 "", 2248 []>, Sched<[]> { 2249 bits<5> Zdn; 2250 bits<5> Zm; 2251 bits<3> Pg; 2252 bit imm; 2253 let Inst{31-24} = 0b01100100; 2254 let Inst{23-22} = sz; 2255 let Inst{21-17} = 0; 2256 let Inst{16} = imm; 2257 let Inst{15-13} = 0b100; 2258 let Inst{12-10} = Pg; 2259 let Inst{9-5} = Zm; 2260 let Inst{4-0} = Zdn; 2261 2262 let Constraints = "$Zdn = $_Zdn"; 2263 let DestructiveInstType = DestructiveOther; 2264 let ElementSize = zprty.ElementSize; 2265} 2266 2267multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2268 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2269 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2270 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2271 2272 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2273 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2274 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2275 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2276 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2277 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2278} 2279 2280//===----------------------------------------------------------------------===// 2281// SVE2 Floating Point Convert Group 2282//===----------------------------------------------------------------------===// 2283 2284class sve2_fp_convert_precision<bits<4> opc, string asm, 2285 ZPRRegOp zprty1, ZPRRegOp zprty2> 2286: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2287 asm, "\t$Zd, $Pg/m, $Zn", 2288 "", 2289 []>, Sched<[]> { 2290 bits<5> Zd; 2291 bits<5> Zn; 2292 bits<3> Pg; 2293 let Inst{31-24} = 0b01100100; 2294 let Inst{23-22} = opc{3-2}; 2295 let Inst{21-18} = 0b0010; 2296 let Inst{17-16} = opc{1-0}; 2297 let Inst{15-13} = 0b101; 2298 let Inst{12-10} = Pg; 2299 let Inst{9-5} = Zn; 2300 let Inst{4-0} = Zd; 2301 2302 let Constraints = "$Zd = $_Zd"; 2303} 2304 2305multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2306 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2307 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2308 2309 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2310 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2311} 2312 2313multiclass sve2_fp_convert_up_long<string asm, string op> { 2314 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2315 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2316 2317 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2318 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2319} 2320 2321multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2322 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2323 2324 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2325} 2326 2327//===----------------------------------------------------------------------===// 2328// SVE2 Floating Point Pairwise Group 2329//===----------------------------------------------------------------------===// 2330 2331class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2332 ZPRRegOp zprty> 2333: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2334 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2335 "", 2336 []>, Sched<[]> { 2337 bits<3> Pg; 2338 bits<5> Zm; 2339 bits<5> Zdn; 2340 let Inst{31-24} = 0b01100100; 2341 let Inst{23-22} = sz; 2342 let Inst{21-19} = 0b010; 2343 let Inst{18-16} = opc; 2344 let Inst{15-13} = 0b100; 2345 let Inst{12-10} = Pg; 2346 let Inst{9-5} = Zm; 2347 let Inst{4-0} = Zdn; 2348 2349 let Constraints = "$Zdn = $_Zdn"; 2350 let DestructiveInstType = DestructiveOther; 2351 let ElementSize = zprty.ElementSize; 2352} 2353 2354multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2355 SDPatternOperator op> { 2356 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2357 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2358 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2359 2360 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2361 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2362 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2363} 2364 2365//===----------------------------------------------------------------------===// 2366// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2367//===----------------------------------------------------------------------===// 2368 2369class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2370: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2371 VectorIndexH32b:$iop), 2372 asm, "\t$Zda, $Zn, $Zm$iop", 2373 "", 2374 []>, Sched<[]> { 2375 bits<5> Zda; 2376 bits<5> Zn; 2377 bits<3> Zm; 2378 bits<3> iop; 2379 let Inst{31-21} = 0b01100100101; 2380 let Inst{20-19} = iop{2-1}; 2381 let Inst{18-16} = Zm; 2382 let Inst{15-14} = 0b01; 2383 let Inst{13} = opc{1}; 2384 let Inst{12} = 0b0; 2385 let Inst{11} = iop{0}; 2386 let Inst{10} = opc{0}; 2387 let Inst{9-5} = Zn; 2388 let Inst{4-0} = Zda; 2389 2390 let Constraints = "$Zda = $_Zda"; 2391 let DestructiveInstType = DestructiveOther; 2392 let ElementSize = ElementSizeNone; 2393} 2394 2395multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2396 SDPatternOperator op> { 2397 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2398 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2399} 2400 2401//===----------------------------------------------------------------------===// 2402// SVE2 Floating Point Widening Multiply-Add Group 2403//===----------------------------------------------------------------------===// 2404 2405class sve2_fp_mla_long<bits<2> opc, string asm> 2406: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2407 asm, "\t$Zda, $Zn, $Zm", 2408 "", 2409 []>, Sched<[]> { 2410 bits<5> Zda; 2411 bits<5> Zn; 2412 bits<5> Zm; 2413 let Inst{31-21} = 0b01100100101; 2414 let Inst{20-16} = Zm; 2415 let Inst{15-14} = 0b10; 2416 let Inst{13} = opc{1}; 2417 let Inst{12-11} = 0b00; 2418 let Inst{10} = opc{0}; 2419 let Inst{9-5} = Zn; 2420 let Inst{4-0} = Zda; 2421 2422 let Constraints = "$Zda = $_Zda"; 2423 let DestructiveInstType = DestructiveOther; 2424 let ElementSize = ElementSizeNone; 2425} 2426 2427multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2428 def NAME : sve2_fp_mla_long<opc, asm>; 2429 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2430} 2431 2432//===----------------------------------------------------------------------===// 2433// SVE Stack Allocation Group 2434//===----------------------------------------------------------------------===// 2435 2436class sve_int_arith_vl<bit opc, string asm> 2437: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2438 asm, "\t$Rd, $Rn, $imm6", 2439 "", 2440 []>, Sched<[]> { 2441 bits<5> Rd; 2442 bits<5> Rn; 2443 bits<6> imm6; 2444 let Inst{31-23} = 0b000001000; 2445 let Inst{22} = opc; 2446 let Inst{21} = 0b1; 2447 let Inst{20-16} = Rn; 2448 let Inst{15-11} = 0b01010; 2449 let Inst{10-5} = imm6; 2450 let Inst{4-0} = Rd; 2451} 2452 2453class sve_int_read_vl_a<bit op, bits<5> opc2, string asm> 2454: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2455 asm, "\t$Rd, $imm6", 2456 "", 2457 []>, Sched<[]> { 2458 bits<5> Rd; 2459 bits<6> imm6; 2460 let Inst{31-23} = 0b000001001; 2461 let Inst{22} = op; 2462 let Inst{21} = 0b1; 2463 let Inst{20-16} = opc2{4-0}; 2464 let Inst{15-11} = 0b01010; 2465 let Inst{10-5} = imm6; 2466 let Inst{4-0} = Rd; 2467} 2468 2469//===----------------------------------------------------------------------===// 2470// SVE Permute - In Lane Group 2471//===----------------------------------------------------------------------===// 2472 2473class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2474 ZPRRegOp zprty> 2475: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2476 asm, "\t$Zd, $Zn, $Zm", 2477 "", 2478 []>, Sched<[]> { 2479 bits<5> Zd; 2480 bits<5> Zm; 2481 bits<5> Zn; 2482 let Inst{31-24} = 0b00000101; 2483 let Inst{23-22} = sz8_64; 2484 let Inst{21} = 0b1; 2485 let Inst{20-16} = Zm; 2486 let Inst{15-13} = 0b011; 2487 let Inst{12-10} = opc; 2488 let Inst{9-5} = Zn; 2489 let Inst{4-0} = Zd; 2490} 2491 2492multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2493 SDPatternOperator op> { 2494 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2495 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2496 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2497 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2498 2499 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2500 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2501 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2502 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2503 2504 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2505 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2506 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2507 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2508 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2509 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2510 2511 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2512} 2513 2514//===----------------------------------------------------------------------===// 2515// SVE Floating Point Unary Operations Group 2516//===----------------------------------------------------------------------===// 2517 2518class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2519 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2520: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2521 asm, "\t$Zd, $Pg/m, $Zn", 2522 "", 2523 []>, Sched<[]> { 2524 bits<3> Pg; 2525 bits<5> Zd; 2526 bits<5> Zn; 2527 let Inst{31-24} = 0b01100101; 2528 let Inst{23-22} = opc{6-5}; 2529 let Inst{21} = 0b0; 2530 let Inst{20-16} = opc{4-0}; 2531 let Inst{15-13} = 0b101; 2532 let Inst{12-10} = Pg; 2533 let Inst{9-5} = Zn; 2534 let Inst{4-0} = Zd; 2535 2536 let Constraints = "$Zd = $_Zd"; 2537 let DestructiveInstType = DestructiveUnaryPassthru; 2538 let ElementSize = Sz; 2539} 2540 2541multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2542 RegisterOperand i_zprtype, 2543 RegisterOperand o_zprtype, 2544 SDPatternOperator int_op, 2545 SDPatternOperator ir_op, ValueType vt1, 2546 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2547 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2548 2549 // convert vt1 to a packed type for the intrinsic patterns 2550 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2551 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2552 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2553 1 : vt1); 2554 2555 // convert vt3 to a packed type for the intrinsic patterns 2556 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2557 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2558 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2559 1 : vt3); 2560 2561 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2562 2563 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2564} 2565 2566multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2567 RegisterOperand i_zprtype, 2568 RegisterOperand o_zprtype, 2569 SDPatternOperator int_op, 2570 SDPatternOperator ir_op, ValueType vt1, 2571 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2572 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2573 2574 // convert vt1 to a packed type for the intrinsic patterns 2575 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2576 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2577 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2578 1 : vt1); 2579 2580 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2581 2582 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2583} 2584 2585multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2586 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>, 2587 SVEPseudo2Instr<NAME # _H, 1>; 2588 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>, 2589 SVEPseudo2Instr<NAME # _S, 1>; 2590 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>, 2591 SVEPseudo2Instr<NAME # _D, 1>; 2592 2593 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2594 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2595 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2596 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2597 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2598 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2599 2600 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 2601 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 2602 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 2603 2604 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2605 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2606 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2607 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2608 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2609 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _UNDEF_D)>; 2610} 2611 2612multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2613 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2614 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2615 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2616 2617 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2618 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2619 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2620} 2621 2622multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2623 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2624 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2625} 2626 2627//===----------------------------------------------------------------------===// 2628// SVE Floating Point Unary Operations - Unpredicated Group 2629//===----------------------------------------------------------------------===// 2630 2631class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2632 ZPRRegOp zprty> 2633: I<(outs zprty:$Zd), (ins zprty:$Zn), 2634 asm, "\t$Zd, $Zn", 2635 "", 2636 []>, Sched<[]> { 2637 bits<5> Zd; 2638 bits<5> Zn; 2639 let Inst{31-24} = 0b01100101; 2640 let Inst{23-22} = sz; 2641 let Inst{21-19} = 0b001; 2642 let Inst{18-16} = opc; 2643 let Inst{15-10} = 0b001100; 2644 let Inst{9-5} = Zn; 2645 let Inst{4-0} = Zd; 2646} 2647 2648multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2649 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2650 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2651 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2652 2653 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2654 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2655 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2656} 2657 2658//===----------------------------------------------------------------------===// 2659// SVE Integer Arithmetic - Binary Predicated Group 2660//===----------------------------------------------------------------------===// 2661 2662class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2663 string asm, ZPRRegOp zprty> 2664: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2665 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2666 bits<3> Pg; 2667 bits<5> Zdn; 2668 bits<5> Zm; 2669 let Inst{31-24} = 0b00000100; 2670 let Inst{23-22} = sz8_64; 2671 let Inst{21} = 0b0; 2672 let Inst{20-19} = fmt; 2673 let Inst{18-16} = opc; 2674 let Inst{15-13} = 0b000; 2675 let Inst{12-10} = Pg; 2676 let Inst{9-5} = Zm; 2677 let Inst{4-0} = Zdn; 2678 2679 let Constraints = "$Zdn = $_Zdn"; 2680 let DestructiveInstType = DestructiveOther; 2681 let ElementSize = zprty.ElementSize; 2682} 2683 2684multiclass sve_int_bin_pred_log<bits<3> opc, string asm, SDPatternOperator op> { 2685 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>; 2686 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>; 2687 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>; 2688 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>; 2689 2690 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2691 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2692 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2693 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2694} 2695 2696multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2697 SDPatternOperator op, 2698 DestructiveInstTypeEnum flags, 2699 string revname="", bit isReverseInstr=0> { 2700 let DestructiveInstType = flags in { 2701 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2702 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2703 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2704 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2705 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2706 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2707 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2708 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2709 } 2710 2711 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2712 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2713 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2714 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2715} 2716 2717multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2718 SDPatternOperator op, 2719 DestructiveInstTypeEnum flags> { 2720 let DestructiveInstType = flags in { 2721 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2722 SVEPseudo2Instr<Ps # _B, 1>; 2723 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2724 SVEPseudo2Instr<Ps # _H, 1>; 2725 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2726 SVEPseudo2Instr<Ps # _S, 1>; 2727 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2728 SVEPseudo2Instr<Ps # _D, 1>; 2729 } 2730 2731 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2732 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2733 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2734 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2735} 2736 2737multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2738 SDPatternOperator op, 2739 DestructiveInstTypeEnum flags> { 2740 let DestructiveInstType = flags in { 2741 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2742 SVEPseudo2Instr<Ps # _B, 1>; 2743 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2744 SVEPseudo2Instr<Ps # _H, 1>; 2745 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2746 SVEPseudo2Instr<Ps # _S, 1>; 2747 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2748 SVEPseudo2Instr<Ps # _D, 1>; 2749 } 2750 2751 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2752 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2753 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2754 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2755} 2756 2757// Special case for divides which are not defined for 8b/16b elements. 2758multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2759 SDPatternOperator op, 2760 DestructiveInstTypeEnum flags, 2761 string revname="", bit isReverseInstr=0> { 2762 let DestructiveInstType = flags in { 2763 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2764 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2765 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2766 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2767 } 2768 2769 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2770 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2771} 2772 2773//===----------------------------------------------------------------------===// 2774// SVE Integer Multiply-Add Group 2775//===----------------------------------------------------------------------===// 2776 2777class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2778 ZPRRegOp zprty> 2779: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2780 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2781 "", 2782 []>, Sched<[]> { 2783 bits<3> Pg; 2784 bits<5> Zdn; 2785 bits<5> Za; 2786 bits<5> Zm; 2787 let Inst{31-24} = 0b00000100; 2788 let Inst{23-22} = sz8_64; 2789 let Inst{21} = 0b0; 2790 let Inst{20-16} = Zm; 2791 let Inst{15-14} = 0b11; 2792 let Inst{13} = opc; 2793 let Inst{12-10} = Pg; 2794 let Inst{9-5} = Za; 2795 let Inst{4-0} = Zdn; 2796 2797 let Constraints = "$Zdn = $_Zdn"; 2798 let DestructiveInstType = DestructiveOther; 2799 let ElementSize = zprty.ElementSize; 2800} 2801 2802multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2803 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2804 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2805 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2806 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2807 2808 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2809 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2810 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2811 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2812} 2813 2814class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2815 ZPRRegOp zprty> 2816: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2817 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2818 "", 2819 []>, Sched<[]> { 2820 bits<3> Pg; 2821 bits<5> Zda; 2822 bits<5> Zm; 2823 bits<5> Zn; 2824 let Inst{31-24} = 0b00000100; 2825 let Inst{23-22} = sz8_64; 2826 let Inst{21} = 0b0; 2827 let Inst{20-16} = Zm; 2828 let Inst{15-14} = 0b01; 2829 let Inst{13} = opc; 2830 let Inst{12-10} = Pg; 2831 let Inst{9-5} = Zn; 2832 let Inst{4-0} = Zda; 2833 2834 let Constraints = "$Zda = $_Zda"; 2835 let DestructiveInstType = DestructiveOther; 2836 let ElementSize = zprty.ElementSize; 2837} 2838 2839multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op, 2840 SDPatternOperator outerop, SDPatternOperator mulop> { 2841 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2842 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2843 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2844 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2845 2846 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2847 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2848 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2849 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2850 2851 def : Pat<(outerop nxv16i8:$Op1, (mulop nxv16i1:$pred, nxv16i8:$Op2, nxv16i8:$Op3)), 2852 (!cast<Instruction>(NAME # _B) $pred, $Op1, $Op2, $Op3)>; 2853 def : Pat<(outerop nxv8i16:$Op1, (mulop nxv8i1:$pred, nxv8i16:$Op2, nxv8i16:$Op3)), 2854 (!cast<Instruction>(NAME # _H) $pred, $Op1, $Op2, $Op3)>; 2855 def : Pat<(outerop nxv4i32:$Op1, (mulop nxv4i1:$pred, nxv4i32:$Op2, nxv4i32:$Op3)), 2856 (!cast<Instruction>(NAME # _S) $pred, $Op1, $Op2, $Op3)>; 2857 def : Pat<(outerop nxv2i64:$Op1, (mulop nxv2i1:$pred, nxv2i64:$Op2, nxv2i64:$Op3)), 2858 (!cast<Instruction>(NAME # _D) $pred, $Op1, $Op2, $Op3)>; 2859} 2860 2861//===----------------------------------------------------------------------===// 2862// SVE2 Integer Multiply-Add - Unpredicated Group 2863//===----------------------------------------------------------------------===// 2864 2865class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2866 ZPRRegOp zprty1, ZPRRegOp zprty2> 2867: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2868 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2869 bits<5> Zda; 2870 bits<5> Zn; 2871 bits<5> Zm; 2872 let Inst{31-24} = 0b01000100; 2873 let Inst{23-22} = sz; 2874 let Inst{21} = 0b0; 2875 let Inst{20-16} = Zm; 2876 let Inst{15} = 0b0; 2877 let Inst{14-10} = opc; 2878 let Inst{9-5} = Zn; 2879 let Inst{4-0} = Zda; 2880 2881 let Constraints = "$Zda = $_Zda"; 2882 let DestructiveInstType = DestructiveOther; 2883 let ElementSize = ElementSizeNone; 2884} 2885 2886multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 2887 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 2888 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 2889 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 2890 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 2891 2892 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2893 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2894 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2895 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2896} 2897 2898multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 2899 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 2900 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 2901 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 2902 2903 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 2904 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 2905 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 2906} 2907 2908//===----------------------------------------------------------------------===// 2909// SVE2 Integer Multiply-Add - Indexed Group 2910//===----------------------------------------------------------------------===// 2911 2912class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 2913 ZPRRegOp zprty1, ZPRRegOp zprty2, 2914 ZPRRegOp zprty3, Operand itype> 2915: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2916 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2917 bits<5> Zda; 2918 bits<5> Zn; 2919 let Inst{31-24} = 0b01000100; 2920 let Inst{23-22} = sz; 2921 let Inst{21} = 0b1; 2922 let Inst{15-10} = opc; 2923 let Inst{9-5} = Zn; 2924 let Inst{4-0} = Zda; 2925 2926 let Constraints = "$Zda = $_Zda"; 2927 let DestructiveInstType = DestructiveOther; 2928 let ElementSize = ElementSizeNone; 2929} 2930 2931multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 2932 SDPatternOperator op> { 2933 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 2934 bits<3> Zm; 2935 bits<3> iop; 2936 let Inst{22} = iop{2}; 2937 let Inst{20-19} = iop{1-0}; 2938 let Inst{18-16} = Zm; 2939 } 2940 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 2941 bits<3> Zm; 2942 bits<2> iop; 2943 let Inst{20-19} = iop; 2944 let Inst{18-16} = Zm; 2945 } 2946 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 2947 bits<4> Zm; 2948 bit iop; 2949 let Inst{20} = iop; 2950 let Inst{19-16} = Zm; 2951 } 2952 2953 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 2954 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 2955 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 2956} 2957 2958//===----------------------------------------------------------------------===// 2959// SVE2 Integer Multiply-Add Long - Indexed Group 2960//===----------------------------------------------------------------------===// 2961 2962multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 2963 SDPatternOperator op> { 2964 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2965 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 2966 bits<3> Zm; 2967 bits<3> iop; 2968 let Inst{20-19} = iop{2-1}; 2969 let Inst{18-16} = Zm; 2970 let Inst{11} = iop{0}; 2971 } 2972 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2973 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 2974 bits<4> Zm; 2975 bits<2> iop; 2976 let Inst{20} = iop{1}; 2977 let Inst{19-16} = Zm; 2978 let Inst{11} = iop{0}; 2979 } 2980 2981 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 2982 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 2983} 2984 2985//===----------------------------------------------------------------------===// 2986// SVE Integer Dot Product Group 2987//===----------------------------------------------------------------------===// 2988 2989class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 2990 ZPRRegOp zprty2> 2991: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 2992 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2993 bits<5> Zda; 2994 bits<5> Zn; 2995 bits<5> Zm; 2996 let Inst{31-23} = 0b010001001; 2997 let Inst{22} = sz; 2998 let Inst{21} = 0; 2999 let Inst{20-16} = Zm; 3000 let Inst{15-11} = 0; 3001 let Inst{10} = U; 3002 let Inst{9-5} = Zn; 3003 let Inst{4-0} = Zda; 3004 3005 let Constraints = "$Zda = $_Zda"; 3006 let DestructiveInstType = DestructiveOther; 3007} 3008 3009multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 3010 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 3011 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 3012 3013 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 3014 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 3015} 3016 3017//===----------------------------------------------------------------------===// 3018// SVE Integer Dot Product Group - Indexed Group 3019//===----------------------------------------------------------------------===// 3020 3021class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 3022 ZPRRegOp zprty1, ZPRRegOp zprty2, 3023 ZPRRegOp zprty3, Operand itype> 3024: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3025 asm, "\t$Zda, $Zn, $Zm$iop", 3026 "", []>, Sched<[]> { 3027 bits<5> Zda; 3028 bits<5> Zn; 3029 let Inst{31-23} = 0b010001001; 3030 let Inst{22} = sz; 3031 let Inst{21} = 0b1; 3032 let Inst{15-11} = 0; 3033 let Inst{10} = U; 3034 let Inst{9-5} = Zn; 3035 let Inst{4-0} = Zda; 3036 3037 let Constraints = "$Zda = $_Zda"; 3038 let DestructiveInstType = DestructiveOther; 3039} 3040 3041multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 3042 SDPatternOperator op> { 3043 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 3044 bits<2> iop; 3045 bits<3> Zm; 3046 let Inst{20-19} = iop; 3047 let Inst{18-16} = Zm; 3048 } 3049 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 3050 bits<1> iop; 3051 bits<4> Zm; 3052 let Inst{20} = iop; 3053 let Inst{19-16} = Zm; 3054 } 3055 3056 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3057 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3058} 3059 3060//===----------------------------------------------------------------------===// 3061// SVE2 Complex Integer Dot Product Group 3062//===----------------------------------------------------------------------===// 3063 3064class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 3065 ZPRRegOp zprty1, ZPRRegOp zprty2> 3066: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 3067 complexrotateop:$rot), 3068 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 3069 bits<5> Zda; 3070 bits<5> Zn; 3071 bits<5> Zm; 3072 bits<2> rot; 3073 let Inst{31-24} = 0b01000100; 3074 let Inst{23-22} = sz; 3075 let Inst{21} = 0b0; 3076 let Inst{20-16} = Zm; 3077 let Inst{15-12} = opc; 3078 let Inst{11-10} = rot; 3079 let Inst{9-5} = Zn; 3080 let Inst{4-0} = Zda; 3081 3082 let Constraints = "$Zda = $_Zda"; 3083 let DestructiveInstType = DestructiveOther; 3084 let ElementSize = ElementSizeNone; 3085} 3086 3087multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 3088 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 3089 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 3090 3091 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3092 (i32 complexrotateop:$imm))), 3093 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 3094 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3095 (i32 complexrotateop:$imm))), 3096 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 3097} 3098 3099//===----------------------------------------------------------------------===// 3100// SVE2 Complex Multiply-Add Group 3101//===----------------------------------------------------------------------===// 3102 3103multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 3104 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 3105 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 3106 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 3107 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 3108 3109 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 3110 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 3111 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 3112 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 3113} 3114 3115//===----------------------------------------------------------------------===// 3116// SVE2 Complex Integer Dot Product - Indexed Group 3117//===----------------------------------------------------------------------===// 3118 3119class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 3120 ZPRRegOp zprty1, ZPRRegOp zprty2, 3121 ZPRRegOp zprty3, Operand itype> 3122: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 3123 complexrotateop:$rot), 3124 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 3125 bits<5> Zda; 3126 bits<5> Zn; 3127 bits<2> rot; 3128 let Inst{31-24} = 0b01000100; 3129 let Inst{23-22} = sz; 3130 let Inst{21} = 0b1; 3131 let Inst{15-12} = opc; 3132 let Inst{11-10} = rot; 3133 let Inst{9-5} = Zn; 3134 let Inst{4-0} = Zda; 3135 3136 let Constraints = "$Zda = $_Zda"; 3137 let DestructiveInstType = DestructiveOther; 3138 let ElementSize = ElementSizeNone; 3139} 3140 3141multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 3142 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 3143 bits<2> iop; 3144 bits<3> Zm; 3145 let Inst{20-19} = iop; 3146 let Inst{18-16} = Zm; 3147 } 3148 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 3149 bit iop; 3150 bits<4> Zm; 3151 let Inst{20} = iop; 3152 let Inst{19-16} = Zm; 3153 } 3154 3155 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3156 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3157 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3158 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3159 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3160 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3161} 3162 3163//===----------------------------------------------------------------------===// 3164// SVE2 Complex Multiply-Add - Indexed Group 3165//===----------------------------------------------------------------------===// 3166 3167multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 3168 SDPatternOperator op> { 3169 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 3170 bits<2> iop; 3171 bits<3> Zm; 3172 let Inst{20-19} = iop; 3173 let Inst{18-16} = Zm; 3174 } 3175 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 3176 bit iop; 3177 bits<4> Zm; 3178 let Inst{20} = iop; 3179 let Inst{19-16} = Zm; 3180 } 3181 3182 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3183 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3184 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3185 3186 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 3187 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3188 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3189} 3190 3191//===----------------------------------------------------------------------===// 3192// SVE2 Integer Multiply - Unpredicated Group 3193//===----------------------------------------------------------------------===// 3194 3195class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 3196: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 3197 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3198 bits<5> Zd; 3199 bits<5> Zm; 3200 bits<5> Zn; 3201 let Inst{31-24} = 0b00000100; 3202 let Inst{23-22} = sz; 3203 let Inst{21} = 0b1; 3204 let Inst{20-16} = Zm; 3205 let Inst{15-13} = 0b011; 3206 let Inst{12-10} = opc; 3207 let Inst{9-5} = Zn; 3208 let Inst{4-0} = Zd; 3209} 3210 3211multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 3212 SDPatternOperator op_pred = null_frag> { 3213 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3214 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 3215 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 3216 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 3217 3218 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3219 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3220 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3221 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3222 3223 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3224 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3225 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3226 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3227} 3228 3229multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 3230 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3231 3232 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3233} 3234 3235//===----------------------------------------------------------------------===// 3236// SVE2 Integer Multiply - Indexed Group 3237//===----------------------------------------------------------------------===// 3238 3239class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 3240 ZPRRegOp zprty1, ZPRRegOp zprty2, 3241 ZPRRegOp zprty3, Operand itype> 3242: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 3243 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 3244 bits<5> Zd; 3245 bits<5> Zn; 3246 let Inst{31-24} = 0b01000100; 3247 let Inst{23-22} = sz; 3248 let Inst{21} = 0b1; 3249 let Inst{15-14} = 0b11; 3250 let Inst{13-10} = opc; 3251 let Inst{9-5} = Zn; 3252 let Inst{4-0} = Zd; 3253} 3254 3255multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3256 SDPatternOperator op> { 3257 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3258 bits<3> Zm; 3259 bits<3> iop; 3260 let Inst{22} = iop{2}; 3261 let Inst{20-19} = iop{1-0}; 3262 let Inst{18-16} = Zm; 3263 } 3264 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3265 bits<3> Zm; 3266 bits<2> iop; 3267 let Inst{20-19} = iop; 3268 let Inst{18-16} = Zm; 3269 } 3270 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3271 bits<4> Zm; 3272 bit iop; 3273 let Inst{20} = iop; 3274 let Inst{19-16} = Zm; 3275 } 3276 3277 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3278 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3279 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3280} 3281 3282multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3283 SDPatternOperator op> { 3284 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3285 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3286 bits<3> Zm; 3287 bits<3> iop; 3288 let Inst{20-19} = iop{2-1}; 3289 let Inst{18-16} = Zm; 3290 let Inst{11} = iop{0}; 3291 } 3292 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3293 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3294 bits<4> Zm; 3295 bits<2> iop; 3296 let Inst{20} = iop{1}; 3297 let Inst{19-16} = Zm; 3298 let Inst{11} = iop{0}; 3299 } 3300 3301 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3302 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3303} 3304 3305//===----------------------------------------------------------------------===// 3306// SVE2 Integer - Predicated Group 3307//===----------------------------------------------------------------------===// 3308 3309class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3310 ZPRRegOp zprty> 3311: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3312 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3313 bits<3> Pg; 3314 bits<5> Zm; 3315 bits<5> Zdn; 3316 let Inst{31-24} = 0b01000100; 3317 let Inst{23-22} = sz; 3318 let Inst{21-20} = 0b01; 3319 let Inst{20-16} = opc{5-1}; 3320 let Inst{15-14} = 0b10; 3321 let Inst{13} = opc{0}; 3322 let Inst{12-10} = Pg; 3323 let Inst{9-5} = Zm; 3324 let Inst{4-0} = Zdn; 3325 3326 let Constraints = "$Zdn = $_Zdn"; 3327 let DestructiveInstType = DestructiveOther; 3328 let ElementSize = zprty.ElementSize; 3329} 3330 3331multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op, 3332 string Ps = "", 3333 DestructiveInstTypeEnum flags=DestructiveOther, 3334 string revname="", bit isReverseInstr=0> { 3335 let DestructiveInstType = flags in { 3336 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>, 3337 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 3338 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>, 3339 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 3340 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>, 3341 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 3342 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>, 3343 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 3344 } 3345 3346 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3347 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3348 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3349 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3350} 3351 3352class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3353 ZPRRegOp zprty1, ZPRRegOp zprty2> 3354: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3355 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3356 bits<3> Pg; 3357 bits<5> Zn; 3358 bits<5> Zda; 3359 let Inst{31-24} = 0b01000100; 3360 let Inst{23-22} = sz; 3361 let Inst{21-17} = 0b00010; 3362 let Inst{16} = U; 3363 let Inst{15-13} = 0b101; 3364 let Inst{12-10} = Pg; 3365 let Inst{9-5} = Zn; 3366 let Inst{4-0} = Zda; 3367 3368 let Constraints = "$Zda = $_Zda"; 3369 let DestructiveInstType = DestructiveOther; 3370 let ElementSize = zprty1.ElementSize; 3371} 3372 3373multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3374 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3375 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3376 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3377 3378 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3379 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3380 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3381} 3382 3383class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3384 string asm, ZPRRegOp zprty> 3385: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3386 asm, "\t$Zd, $Pg/m, $Zn", 3387 "", 3388 []>, Sched<[]> { 3389 bits<3> Pg; 3390 bits<5> Zd; 3391 bits<5> Zn; 3392 let Inst{31-24} = 0b01000100; 3393 let Inst{23-22} = sz; 3394 let Inst{21-20} = 0b00; 3395 let Inst{19} = Q; 3396 let Inst{18} = 0b0; 3397 let Inst{17-16} = opc; 3398 let Inst{15-13} = 0b101; 3399 let Inst{12-10} = Pg; 3400 let Inst{9-5} = Zn; 3401 let Inst{4-0} = Zd; 3402 3403 let Constraints = "$Zd = $_Zd"; 3404 let DestructiveInstType = DestructiveUnaryPassthru; 3405 let ElementSize = zprty.ElementSize; 3406} 3407 3408multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3409 SDPatternOperator op> { 3410 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3411 SVEPseudo2Instr<NAME # _S, 1>; 3412 3413 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3414 3415 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3416 3417 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3418} 3419 3420multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3421 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>, 3422 SVEPseudo2Instr<NAME # _B, 1>; 3423 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>, 3424 SVEPseudo2Instr<NAME # _H, 1>; 3425 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3426 SVEPseudo2Instr<NAME # _S, 1>; 3427 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>, 3428 SVEPseudo2Instr<NAME # _D, 1>; 3429 3430 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3431 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3432 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3433 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3434 3435 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3436 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3437 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3438 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3439 3440 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3441 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3442 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3443 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3444} 3445 3446//===----------------------------------------------------------------------===// 3447// SVE2 Widening Integer Arithmetic Group 3448//===----------------------------------------------------------------------===// 3449 3450class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3451 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3452: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3453 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3454 bits<5> Zd; 3455 bits<5> Zn; 3456 bits<5> Zm; 3457 let Inst{31-24} = 0b01000101; 3458 let Inst{23-22} = sz; 3459 let Inst{21} = 0b0; 3460 let Inst{20-16} = Zm; 3461 let Inst{15} = 0b0; 3462 let Inst{14-10} = opc; 3463 let Inst{9-5} = Zn; 3464 let Inst{4-0} = Zd; 3465} 3466 3467multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3468 SDPatternOperator op> { 3469 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3470 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3471 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3472 3473 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3474 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3475 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3476} 3477 3478multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3479 SDPatternOperator op> { 3480 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3481 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3482 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3483 3484 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3485 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3486 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3487} 3488 3489multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3490 SDPatternOperator op> { 3491 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3492 3493 // To avoid using 128 bit elements in the IR, the pattern below works with 3494 // llvm intrinsics with the _pair suffix, to reflect that 3495 // _Q is implemented as a pair of _D. 3496 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3497} 3498 3499multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3500 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3501 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3502 3503 // To avoid using 128 bit elements in the IR, the patterns below work with 3504 // llvm intrinsics with the _pair suffix, to reflect that 3505 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3506 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3507 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3508} 3509 3510//===----------------------------------------------------------------------===// 3511// SVE2 Misc Group 3512//===----------------------------------------------------------------------===// 3513 3514class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3515 ZPRRegOp zprty1, ZPRRegOp zprty2> 3516: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3517 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3518 bits<5> Zd; 3519 bits<5> Zn; 3520 bits<5> Zm; 3521 let Inst{31-24} = 0b01000101; 3522 let Inst{23-22} = sz; 3523 let Inst{21} = 0b0; 3524 let Inst{20-16} = Zm; 3525 let Inst{15-14} = 0b10; 3526 let Inst{13-10} = opc; 3527 let Inst{9-5} = Zn; 3528 let Inst{4-0} = Zd; 3529} 3530 3531multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3532 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3533 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3534 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3535 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3536 3537 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3538 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3539 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3540 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3541} 3542 3543multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3544 SDPatternOperator op> { 3545 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3546 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3547 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3548 3549 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3550 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3551 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3552} 3553 3554class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3555 ZPRRegOp zprty1, ZPRRegOp zprty2> 3556: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3557 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3558 bits<5> Zd; 3559 bits<5> Zn; 3560 bits<5> Zm; 3561 let Inst{31-24} = 0b01000101; 3562 let Inst{23-22} = sz; 3563 let Inst{21} = 0b0; 3564 let Inst{20-16} = Zm; 3565 let Inst{15-11} = 0b10010; 3566 let Inst{10} = opc; 3567 let Inst{9-5} = Zn; 3568 let Inst{4-0} = Zd; 3569 3570 let Constraints = "$Zd = $_Zd"; 3571 let DestructiveInstType = DestructiveOther; 3572 let ElementSize = ElementSizeNone; 3573} 3574 3575multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3576 SDPatternOperator op> { 3577 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3578 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3579 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3580 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3581 3582 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3583 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3584 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3585 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3586} 3587 3588class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3589 ZPRRegOp zprty1, ZPRRegOp zprty2, 3590 Operand immtype> 3591: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3592 asm, "\t$Zd, $Zn, $imm", 3593 "", []>, Sched<[]> { 3594 bits<5> Zd; 3595 bits<5> Zn; 3596 bits<5> imm; 3597 let Inst{31-23} = 0b010001010; 3598 let Inst{22} = tsz8_64{2}; 3599 let Inst{21} = 0b0; 3600 let Inst{20-19} = tsz8_64{1-0}; 3601 let Inst{18-16} = imm{2-0}; // imm3 3602 let Inst{15-12} = 0b1010; 3603 let Inst{11-10} = opc; 3604 let Inst{9-5} = Zn; 3605 let Inst{4-0} = Zd; 3606} 3607 3608multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3609 SDPatternOperator op> { 3610 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3611 ZPR16, ZPR8, vecshiftL8>; 3612 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3613 ZPR32, ZPR16, vecshiftL16> { 3614 let Inst{19} = imm{3}; 3615 } 3616 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3617 ZPR64, ZPR32, vecshiftL32> { 3618 let Inst{20-19} = imm{4-3}; 3619 } 3620 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3621 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3622 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3623} 3624 3625//===----------------------------------------------------------------------===// 3626// SVE2 Accumulate Group 3627//===----------------------------------------------------------------------===// 3628 3629class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3630 ZPRRegOp zprty, Operand immtype> 3631: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3632 asm, "\t$Zd, $Zn, $imm", 3633 "", []>, Sched<[]> { 3634 bits<5> Zd; 3635 bits<5> Zn; 3636 bits<6> imm; 3637 let Inst{31-24} = 0b01000101; 3638 let Inst{23-22} = tsz8_64{3-2}; 3639 let Inst{21} = 0b0; 3640 let Inst{20-19} = tsz8_64{1-0}; 3641 let Inst{18-16} = imm{2-0}; // imm3 3642 let Inst{15-11} = 0b11110; 3643 let Inst{10} = opc; 3644 let Inst{9-5} = Zn; 3645 let Inst{4-0} = Zd; 3646 3647 let Constraints = "$Zd = $_Zd"; 3648} 3649 3650multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3651 SDPatternOperator op> { 3652 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3653 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3654 let Inst{19} = imm{3}; 3655 } 3656 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3657 let Inst{20-19} = imm{4-3}; 3658 } 3659 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3660 let Inst{22} = imm{5}; 3661 let Inst{20-19} = imm{4-3}; 3662 } 3663 3664 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3665 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3666 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3667 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3668} 3669 3670multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3671 SDPatternOperator op> { 3672 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3673 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3674 let Inst{19} = imm{3}; 3675 } 3676 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3677 let Inst{20-19} = imm{4-3}; 3678 } 3679 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3680 let Inst{22} = imm{5}; 3681 let Inst{20-19} = imm{4-3}; 3682 } 3683 3684 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3685 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3686 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3687 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3688} 3689 3690class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3691 ZPRRegOp zprty, Operand immtype> 3692: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3693 asm, "\t$Zda, $Zn, $imm", 3694 "", []>, Sched<[]> { 3695 bits<5> Zda; 3696 bits<5> Zn; 3697 bits<6> imm; 3698 let Inst{31-24} = 0b01000101; 3699 let Inst{23-22} = tsz8_64{3-2}; 3700 let Inst{21} = 0b0; 3701 let Inst{20-19} = tsz8_64{1-0}; 3702 let Inst{18-16} = imm{2-0}; // imm3 3703 let Inst{15-12} = 0b1110; 3704 let Inst{11-10} = opc; 3705 let Inst{9-5} = Zn; 3706 let Inst{4-0} = Zda; 3707 3708 let Constraints = "$Zda = $_Zda"; 3709 let DestructiveInstType = DestructiveOther; 3710 let ElementSize = ElementSizeNone; 3711} 3712 3713multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3714 SDPatternOperator op> { 3715 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3716 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3717 let Inst{19} = imm{3}; 3718 } 3719 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3720 let Inst{20-19} = imm{4-3}; 3721 } 3722 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3723 let Inst{22} = imm{5}; 3724 let Inst{20-19} = imm{4-3}; 3725 } 3726 3727 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3728 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3729 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3730 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3731} 3732 3733class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3734: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3735 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3736 bits<5> Zdn; 3737 bits<5> Zm; 3738 bit rot; 3739 let Inst{31-24} = 0b01000101; 3740 let Inst{23-22} = sz; 3741 let Inst{21-17} = 0b00000; 3742 let Inst{16} = opc; 3743 let Inst{15-11} = 0b11011; 3744 let Inst{10} = rot; 3745 let Inst{9-5} = Zm; 3746 let Inst{4-0} = Zdn; 3747 3748 let Constraints = "$Zdn = $_Zdn"; 3749 let DestructiveInstType = DestructiveOther; 3750 let ElementSize = ElementSizeNone; 3751} 3752 3753multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3754 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3755 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3756 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3757 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3758 3759 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3760 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3761 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3762 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3763} 3764 3765class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3766 ZPRRegOp zprty1, ZPRRegOp zprty2> 3767: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3768 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3769 bits<5> Zda; 3770 bits<5> Zn; 3771 bits<5> Zm; 3772 let Inst{31-24} = 0b01000101; 3773 let Inst{23-22} = sz; 3774 let Inst{21} = 0b0; 3775 let Inst{20-16} = Zm; 3776 let Inst{15-14} = 0b11; 3777 let Inst{13-10} = opc; 3778 let Inst{9-5} = Zn; 3779 let Inst{4-0} = Zda; 3780 3781 let Constraints = "$Zda = $_Zda"; 3782 let DestructiveInstType = DestructiveOther; 3783 let ElementSize = ElementSizeNone; 3784} 3785 3786multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3787 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3788 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3789 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3790 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3791 3792 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3793 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3794 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3795 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3796} 3797 3798multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3799 SDPatternOperator op> { 3800 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3801 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3802 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3803 3804 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3805 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3806 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3807} 3808 3809multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3810 SDPatternOperator op> { 3811 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3812 ZPR32, ZPR32>; 3813 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3814 ZPR64, ZPR64>; 3815 3816 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3817 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3818} 3819 3820//===----------------------------------------------------------------------===// 3821// SVE2 Narrowing Group 3822//===----------------------------------------------------------------------===// 3823 3824class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3825 string asm, ZPRRegOp zprty1, 3826 ZPRRegOp zprty2, Operand immtype> 3827: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3828 asm, "\t$Zd, $Zn, $imm", 3829 "", []>, Sched<[]> { 3830 bits<5> Zd; 3831 bits<5> Zn; 3832 bits<5> imm; 3833 let Inst{31-23} = 0b010001010; 3834 let Inst{22} = tsz8_64{2}; 3835 let Inst{21} = 0b1; 3836 let Inst{20-19} = tsz8_64{1-0}; 3837 let Inst{18-16} = imm{2-0}; // imm3 3838 let Inst{15-14} = 0b00; 3839 let Inst{13-11} = opc; 3840 let Inst{10} = 0b0; 3841 let Inst{9-5} = Zn; 3842 let Inst{4-0} = Zd; 3843} 3844 3845multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3846 SDPatternOperator op> { 3847 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3848 tvecshiftR8>; 3849 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3850 tvecshiftR16> { 3851 let Inst{19} = imm{3}; 3852 } 3853 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3854 tvecshiftR32> { 3855 let Inst{20-19} = imm{4-3}; 3856 } 3857 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3858 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3859 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3860} 3861 3862class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3863 string asm, ZPRRegOp zprty1, 3864 ZPRRegOp zprty2, Operand immtype> 3865: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3866 asm, "\t$Zd, $Zn, $imm", 3867 "", []>, Sched<[]> { 3868 bits<5> Zd; 3869 bits<5> Zn; 3870 bits<5> imm; 3871 let Inst{31-23} = 0b010001010; 3872 let Inst{22} = tsz8_64{2}; 3873 let Inst{21} = 0b1; 3874 let Inst{20-19} = tsz8_64{1-0}; 3875 let Inst{18-16} = imm{2-0}; // imm3 3876 let Inst{15-14} = 0b00; 3877 let Inst{13-11} = opc; 3878 let Inst{10} = 0b1; 3879 let Inst{9-5} = Zn; 3880 let Inst{4-0} = Zd; 3881 3882 let Constraints = "$Zd = $_Zd"; 3883} 3884 3885multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 3886 SDPatternOperator op> { 3887 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 3888 tvecshiftR8>; 3889 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 3890 tvecshiftR16> { 3891 let Inst{19} = imm{3}; 3892 } 3893 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 3894 tvecshiftR32> { 3895 let Inst{20-19} = imm{4-3}; 3896 } 3897 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3898 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3899 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3900} 3901 3902class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 3903 ZPRRegOp zprty1, ZPRRegOp zprty2> 3904: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3905 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3906 bits<5> Zd; 3907 bits<5> Zn; 3908 bits<5> Zm; 3909 let Inst{31-24} = 0b01000101; 3910 let Inst{23-22} = sz; 3911 let Inst{21} = 0b1; 3912 let Inst{20-16} = Zm; 3913 let Inst{15-13} = 0b011; 3914 let Inst{12-11} = opc; // S, R 3915 let Inst{10} = 0b0; // Top 3916 let Inst{9-5} = Zn; 3917 let Inst{4-0} = Zd; 3918} 3919 3920multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 3921 SDPatternOperator op> { 3922 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 3923 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 3924 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 3925 3926 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3927 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3928 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3929} 3930 3931class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 3932 ZPRRegOp zprty1, ZPRRegOp zprty2> 3933: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3934 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3935 bits<5> Zd; 3936 bits<5> Zn; 3937 bits<5> Zm; 3938 let Inst{31-24} = 0b01000101; 3939 let Inst{23-22} = sz; 3940 let Inst{21} = 0b1; 3941 let Inst{20-16} = Zm; 3942 let Inst{15-13} = 0b011; 3943 let Inst{12-11} = opc; // S, R 3944 let Inst{10} = 0b1; // Top 3945 let Inst{9-5} = Zn; 3946 let Inst{4-0} = Zd; 3947 3948 let Constraints = "$Zd = $_Zd"; 3949} 3950 3951multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 3952 SDPatternOperator op> { 3953 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 3954 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 3955 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 3956 3957 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3958 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3959 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3960} 3961 3962class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 3963 ZPRRegOp zprty1, ZPRRegOp zprty2> 3964: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 3965 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3966 bits<5> Zd; 3967 bits<5> Zn; 3968 let Inst{31-23} = 0b010001010; 3969 let Inst{22} = tsz8_64{2}; 3970 let Inst{21} = 0b1; 3971 let Inst{20-19} = tsz8_64{1-0}; 3972 let Inst{18-13} = 0b000010; 3973 let Inst{12-11} = opc; 3974 let Inst{10} = 0b0; 3975 let Inst{9-5} = Zn; 3976 let Inst{4-0} = Zd; 3977} 3978 3979multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 3980 SDPatternOperator op> { 3981 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 3982 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 3983 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 3984 3985 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 3986 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 3987 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 3988} 3989 3990class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 3991 ZPRRegOp zprty1, ZPRRegOp zprty2> 3992: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 3993 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3994 bits<5> Zd; 3995 bits<5> Zn; 3996 let Inst{31-23} = 0b010001010; 3997 let Inst{22} = tsz8_64{2}; 3998 let Inst{21} = 0b1; 3999 let Inst{20-19} = tsz8_64{1-0}; 4000 let Inst{18-13} = 0b000010; 4001 let Inst{12-11} = opc; 4002 let Inst{10} = 0b1; 4003 let Inst{9-5} = Zn; 4004 let Inst{4-0} = Zd; 4005 4006 let Constraints = "$Zd = $_Zd"; 4007} 4008 4009multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 4010 SDPatternOperator op> { 4011 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 4012 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 4013 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 4014 4015 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 4016 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 4017 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4018} 4019 4020//===----------------------------------------------------------------------===// 4021// SVE Integer Arithmetic - Unary Predicated Group 4022//===----------------------------------------------------------------------===// 4023 4024class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 4025 string asm, ZPRRegOp zprty> 4026: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 4027 asm, "\t$Zd, $Pg/m, $Zn", 4028 "", 4029 []>, Sched<[]> { 4030 bits<3> Pg; 4031 bits<5> Zd; 4032 bits<5> Zn; 4033 let Inst{31-24} = 0b00000100; 4034 let Inst{23-22} = sz8_64; 4035 let Inst{21-20} = 0b01; 4036 let Inst{19} = opc{0}; 4037 let Inst{18-16} = opc{3-1}; 4038 let Inst{15-13} = 0b101; 4039 let Inst{12-10} = Pg; 4040 let Inst{9-5} = Zn; 4041 let Inst{4-0} = Zd; 4042 4043 let Constraints = "$Zd = $_Zd"; 4044 let DestructiveInstType = DestructiveUnaryPassthru; 4045 let ElementSize = zprty.ElementSize; 4046} 4047 4048multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 4049 SDPatternOperator op> { 4050 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>, 4051 SVEPseudo2Instr<NAME # _B, 1>; 4052 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4053 SVEPseudo2Instr<NAME # _H, 1>; 4054 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4055 SVEPseudo2Instr<NAME # _S, 1>; 4056 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4057 SVEPseudo2Instr<NAME # _D, 1>; 4058 4059 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4060 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4061 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4062 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4063 4064 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4065 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4066 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4067 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4068 4069 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4070 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4071 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4072 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4073} 4074 4075multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 4076 SDPatternOperator op> { 4077 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4078 SVEPseudo2Instr<NAME # _H, 1>; 4079 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4080 SVEPseudo2Instr<NAME # _S, 1>; 4081 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4082 SVEPseudo2Instr<NAME # _D, 1>; 4083 4084 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 4085 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 4086 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 4087 4088 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4089 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4090 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4091 4092 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _UNDEF_H)>; 4093 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _UNDEF_S)>; 4094 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _UNDEF_D)>; 4095} 4096 4097multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 4098 SDPatternOperator op> { 4099 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4100 SVEPseudo2Instr<NAME # _S, 1>; 4101 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4102 SVEPseudo2Instr<NAME # _D, 1>; 4103 4104 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 4105 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 4106 4107 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4108 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4109 4110 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _UNDEF_S)>; 4111 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _UNDEF_D)>; 4112} 4113 4114multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 4115 SDPatternOperator op> { 4116 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4117 SVEPseudo2Instr<NAME # _D, 1>; 4118 4119 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 4120 4121 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4122 4123 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _UNDEF_D)>; 4124} 4125 4126multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 4127 SDPatternOperator op> { 4128 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>, 4129 SVEPseudo2Instr<NAME # _B, 1>; 4130 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4131 SVEPseudo2Instr<NAME # _H, 1>; 4132 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4133 SVEPseudo2Instr<NAME # _S, 1>; 4134 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4135 SVEPseudo2Instr<NAME # _D, 1>; 4136 4137 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4138 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4139 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4140 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4141 4142 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4143 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4144 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4145 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4146 4147 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4148 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4149 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4150 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4151} 4152 4153multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 4154 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4155 SVEPseudo2Instr<NAME # _H, 1>; 4156 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4157 SVEPseudo2Instr<NAME # _S, 1>; 4158 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4159 SVEPseudo2Instr<NAME # _D, 1>; 4160 4161 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4162 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4163 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4164 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4165 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4166 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4167 4168 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4169 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4170 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4171 4172 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4173 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4174 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4175 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4176 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4177 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4178} 4179 4180//===----------------------------------------------------------------------===// 4181// SVE Integer Wide Immediate - Unpredicated Group 4182//===----------------------------------------------------------------------===// 4183class sve_int_dup_imm<bits<2> sz8_64, string asm, 4184 ZPRRegOp zprty, Operand immtype> 4185: I<(outs zprty:$Zd), (ins immtype:$imm), 4186 asm, "\t$Zd, $imm", 4187 "", 4188 []>, Sched<[]> { 4189 bits<5> Zd; 4190 bits<9> imm; 4191 let Inst{31-24} = 0b00100101; 4192 let Inst{23-22} = sz8_64; 4193 let Inst{21-14} = 0b11100011; 4194 let Inst{13} = imm{8}; // sh 4195 let Inst{12-5} = imm{7-0}; // imm8 4196 let Inst{4-0} = Zd; 4197 4198 let isReMaterializable = 1; 4199} 4200 4201multiclass sve_int_dup_imm<string asm> { 4202 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 4203 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 4204 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 4205 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 4206 4207 def : InstAlias<"mov $Zd, $imm", 4208 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 4209 def : InstAlias<"mov $Zd, $imm", 4210 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 4211 def : InstAlias<"mov $Zd, $imm", 4212 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 4213 def : InstAlias<"mov $Zd, $imm", 4214 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 4215 4216 def : InstAlias<"fmov $Zd, #0.0", 4217 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 4218 def : InstAlias<"fmov $Zd, #0.0", 4219 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 4220 def : InstAlias<"fmov $Zd, #0.0", 4221 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 4222} 4223 4224class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 4225 string asm, ZPRRegOp zprty> 4226: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 4227 asm, "\t$Zd, $imm8", 4228 "", 4229 []>, Sched<[]> { 4230 bits<5> Zd; 4231 bits<8> imm8; 4232 let Inst{31-24} = 0b00100101; 4233 let Inst{23-22} = sz8_64; 4234 let Inst{21-14} = 0b11100111; 4235 let Inst{13} = 0b0; 4236 let Inst{12-5} = imm8; 4237 let Inst{4-0} = Zd; 4238 4239 let isReMaterializable = 1; 4240} 4241 4242multiclass sve_int_dup_fpimm<string asm> { 4243 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 4244 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 4245 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 4246 4247 def : InstAlias<"fmov $Zd, $imm8", 4248 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 4249 def : InstAlias<"fmov $Zd, $imm8", 4250 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 4251 def : InstAlias<"fmov $Zd, $imm8", 4252 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 4253} 4254 4255class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 4256 ZPRRegOp zprty, Operand immtype> 4257: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4258 asm, "\t$Zdn, $_Zdn, $imm", 4259 "", 4260 []>, Sched<[]> { 4261 bits<5> Zdn; 4262 bits<9> imm; 4263 let Inst{31-24} = 0b00100101; 4264 let Inst{23-22} = sz8_64; 4265 let Inst{21-19} = 0b100; 4266 let Inst{18-16} = opc; 4267 let Inst{15-14} = 0b11; 4268 let Inst{13} = imm{8}; // sh 4269 let Inst{12-5} = imm{7-0}; // imm8 4270 let Inst{4-0} = Zdn; 4271 4272 let Constraints = "$Zdn = $_Zdn"; 4273 let DestructiveInstType = DestructiveOther; 4274 let ElementSize = ElementSizeNone; 4275} 4276 4277multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { 4278 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4279 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4280 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4281 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4282 4283 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4284 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4285 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4286 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4287} 4288 4289multiclass sve_int_arith_imm0_subr<bits<3> opc, string asm, SDPatternOperator op> { 4290 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4291 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4292 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4293 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4294 4295 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4296 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4297 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4298 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4299} 4300 4301class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 4302 ZPRRegOp zprty, Operand immtype> 4303: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4304 asm, "\t$Zdn, $_Zdn, $imm", 4305 "", 4306 []>, Sched<[]> { 4307 bits<5> Zdn; 4308 bits<8> imm; 4309 let Inst{31-24} = 0b00100101; 4310 let Inst{23-22} = sz8_64; 4311 let Inst{21-16} = opc; 4312 let Inst{15-13} = 0b110; 4313 let Inst{12-5} = imm; 4314 let Inst{4-0} = Zdn; 4315 4316 let Constraints = "$Zdn = $_Zdn"; 4317 let DestructiveInstType = DestructiveOther; 4318 let ElementSize = ElementSizeNone; 4319} 4320 4321multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 4322 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 4323 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 4324 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 4325 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 4326 4327 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 4328 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 4329 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 4330 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 4331} 4332 4333multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4334 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4335 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4336 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4337 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4338 4339 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4340 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4341 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4342 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4343} 4344 4345multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4346 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 4347 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 4348 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 4349 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 4350 4351 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 4352 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 4353 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 4354 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 4355} 4356 4357//===----------------------------------------------------------------------===// 4358// SVE Bitwise Logical - Unpredicated Group 4359//===----------------------------------------------------------------------===// 4360 4361class sve_int_bin_cons_log<bits<2> opc, string asm> 4362: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4363 asm, "\t$Zd, $Zn, $Zm", 4364 "", 4365 []>, Sched<[]> { 4366 bits<5> Zd; 4367 bits<5> Zm; 4368 bits<5> Zn; 4369 let Inst{31-24} = 0b00000100; 4370 let Inst{23-22} = opc{1-0}; 4371 let Inst{21} = 0b1; 4372 let Inst{20-16} = Zm; 4373 let Inst{15-10} = 0b001100; 4374 let Inst{9-5} = Zn; 4375 let Inst{4-0} = Zd; 4376} 4377 4378multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4379 def NAME : sve_int_bin_cons_log<opc, asm>; 4380 4381 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4382 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4383 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4384 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4385 4386 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4387 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4388 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4389 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4390 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4391 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4392} 4393 4394class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4395: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4396 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4397 "", 4398 []>, Sched<[]> { 4399 bits<5> Zdn; 4400 bits<5> Zk; 4401 bits<5> Zm; 4402 let Inst{31-24} = 0b00000100; 4403 let Inst{23-22} = opc{2-1}; 4404 let Inst{21} = 0b1; 4405 let Inst{20-16} = Zm; 4406 let Inst{15-11} = 0b00111; 4407 let Inst{10} = opc{0}; 4408 let Inst{9-5} = Zk; 4409 let Inst{4-0} = Zdn; 4410 4411 let Constraints = "$Zdn = $_Zdn"; 4412 let DestructiveInstType = DestructiveOther; 4413 let ElementSize = ElementSizeNone; 4414} 4415 4416multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op> { 4417 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4418 4419 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4420 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4421 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4422 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4423 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4424 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4425 4426 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4427 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4428 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4429 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4430} 4431 4432class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4433 ZPRRegOp zprty, Operand immtype> 4434: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4435 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4436 "", 4437 []>, Sched<[]> { 4438 bits<5> Zdn; 4439 bits<5> Zm; 4440 bits<6> imm; 4441 let Inst{31-24} = 0b00000100; 4442 let Inst{23-22} = tsz8_64{3-2}; 4443 let Inst{21} = 0b1; 4444 let Inst{20-19} = tsz8_64{1-0}; 4445 let Inst{18-16} = imm{2-0}; // imm3 4446 let Inst{15-10} = 0b001101; 4447 let Inst{9-5} = Zm; 4448 let Inst{4-0} = Zdn; 4449 4450 let Constraints = "$Zdn = $_Zdn"; 4451 let DestructiveInstType = DestructiveOther; 4452 let ElementSize = ElementSizeNone; 4453} 4454 4455multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4456 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4457 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4458 let Inst{19} = imm{3}; 4459 } 4460 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4461 let Inst{20-19} = imm{4-3}; 4462 } 4463 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4464 let Inst{22} = imm{5}; 4465 let Inst{20-19} = imm{4-3}; 4466 } 4467 4468 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4469 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4470 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4471 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4472} 4473 4474//===----------------------------------------------------------------------===// 4475// SVE Integer Wide Immediate - Predicated Group 4476//===----------------------------------------------------------------------===// 4477 4478class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4479 string asm, ZPRRegOp zprty> 4480: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4481 asm, "\t$Zd, $Pg/m, $imm8", 4482 "", 4483 []>, Sched<[]> { 4484 bits<4> Pg; 4485 bits<5> Zd; 4486 bits<8> imm8; 4487 let Inst{31-24} = 0b00000101; 4488 let Inst{23-22} = sz; 4489 let Inst{21-20} = 0b01; 4490 let Inst{19-16} = Pg; 4491 let Inst{15-13} = 0b110; 4492 let Inst{12-5} = imm8; 4493 let Inst{4-0} = Zd; 4494 4495 let Constraints = "$Zd = $_Zd"; 4496 let DestructiveInstType = DestructiveOther; 4497 let ElementSize = zprty.ElementSize; 4498} 4499 4500multiclass sve_int_dup_fpimm_pred<string asm> { 4501 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4502 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4503 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4504 4505 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4506 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4507 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4508 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4509 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4510 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4511} 4512 4513class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4514 ZPRRegOp zprty, string pred_qual, dag iops> 4515: I<(outs zprty:$Zd), iops, 4516 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4517 "", []>, Sched<[]> { 4518 bits<5> Zd; 4519 bits<4> Pg; 4520 bits<9> imm; 4521 let Inst{31-24} = 0b00000101; 4522 let Inst{23-22} = sz8_64; 4523 let Inst{21-20} = 0b01; 4524 let Inst{19-16} = Pg; 4525 let Inst{15} = 0b0; 4526 let Inst{14} = m; 4527 let Inst{13} = imm{8}; // sh 4528 let Inst{12-5} = imm{7-0}; // imm8 4529 let Inst{4-0} = Zd; 4530 4531 let DestructiveInstType = DestructiveOther; 4532 let ElementSize = zprty.ElementSize; 4533} 4534 4535multiclass sve_int_dup_imm_pred_merge_inst< 4536 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4537 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4538 let Constraints = "$Zd = $_Zd" in 4539 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4540 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4541 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4542 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4543 def : Pat<(intty 4544 (vselect predty:$Pg, 4545 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4546 intty:$Zd)), 4547 (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; 4548} 4549 4550multiclass sve_int_dup_imm_pred_merge<string asm> { 4551 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4552 i32, cpy_imm8_opt_lsl_i8>; 4553 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4554 i32, cpy_imm8_opt_lsl_i16>; 4555 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4556 i32, cpy_imm8_opt_lsl_i32>; 4557 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4558 i64, cpy_imm8_opt_lsl_i64>; 4559 4560 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4561 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4562 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4563 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4564 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4565 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4566} 4567 4568multiclass sve_int_dup_imm_pred_zero_inst< 4569 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4570 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4571 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4572 (ins PPRAny:$Pg, cpyimm:$imm)>; 4573 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4574 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4575 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4576 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4577 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4578 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4579 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4580 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4581 def : Pat<(intty 4582 (vselect predty:$Pg, 4583 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4584 (intty (AArch64dup (scalarty 0))))), 4585 (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>; 4586} 4587 4588multiclass sve_int_dup_imm_pred_zero<string asm> { 4589 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4590 i32, cpy_imm8_opt_lsl_i8>; 4591 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4592 i32, cpy_imm8_opt_lsl_i16>; 4593 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4594 i32, cpy_imm8_opt_lsl_i32>; 4595 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4596 i64, cpy_imm8_opt_lsl_i64>; 4597} 4598 4599//===----------------------------------------------------------------------===// 4600// SVE Integer Compare - Vectors Group 4601//===----------------------------------------------------------------------===// 4602 4603class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4604 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4605: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4606 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4607 "", 4608 []>, Sched<[]> { 4609 bits<4> Pd; 4610 bits<3> Pg; 4611 bits<5> Zm; 4612 bits<5> Zn; 4613 let Inst{31-24} = 0b00100100; 4614 let Inst{23-22} = sz8_64; 4615 let Inst{21} = 0b0; 4616 let Inst{20-16} = Zm; 4617 let Inst{15} = opc{2}; 4618 let Inst{14} = cmp_1; 4619 let Inst{13} = opc{1}; 4620 let Inst{12-10} = Pg; 4621 let Inst{9-5} = Zn; 4622 let Inst{4} = opc{0}; 4623 let Inst{3-0} = Pd; 4624 4625 let Defs = [NZCV]; 4626 let ElementSize = pprty.ElementSize; 4627 let isPTestLike = 1; 4628} 4629 4630multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4631 ValueType intvt, Instruction cmp> { 4632 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4633 (cmp $Op1, $Op2, $Op3)>; 4634 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4635 (cmp $Op1, $Op3, $Op2)>; 4636} 4637 4638multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt, 4639 ValueType intvt, Instruction cmp> { 4640 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)), 4641 (cmp $Op1, $Op2)>; 4642 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)), 4643 (cmp $Op1, $Op2)>; 4644} 4645 4646multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4647 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4648 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4649 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4650 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4651 4652 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4653 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4654 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4655 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4656} 4657 4658multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4659 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4660 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4661 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4662 4663 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4664 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4665 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4666} 4667 4668multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4669 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4670 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4671 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4672 4673 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4674 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4675 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4676} 4677 4678 4679//===----------------------------------------------------------------------===// 4680// SVE Integer Compare - Signed Immediate Group 4681//===----------------------------------------------------------------------===// 4682 4683class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4684 ZPRRegOp zprty, 4685 Operand immtype> 4686: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4687 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4688 "", 4689 []>, Sched<[]> { 4690 bits<4> Pd; 4691 bits<3> Pg; 4692 bits<5> Zn; 4693 bits<5> imm5; 4694 let Inst{31-24} = 0b00100101; 4695 let Inst{23-22} = sz8_64; 4696 let Inst{21} = 0b0; 4697 let Inst{20-16} = imm5; 4698 let Inst{15} = opc{2}; 4699 let Inst{14} = 0b0; 4700 let Inst{13} = opc{1}; 4701 let Inst{12-10} = Pg; 4702 let Inst{9-5} = Zn; 4703 let Inst{4} = opc{0}; 4704 let Inst{3-0} = Pd; 4705 4706 let Defs = [NZCV]; 4707 let ElementSize = pprty.ElementSize; 4708 let isPTestLike = 1; 4709} 4710 4711multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4712 ValueType predvt, ValueType intvt, 4713 Operand immtype, Instruction cmp> { 4714 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4715 (intvt ZPR:$Zs1), 4716 (intvt (AArch64dup (immtype:$imm))), 4717 cc)), 4718 (cmp $Pg, $Zs1, immtype:$imm)>; 4719 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4720 (intvt (AArch64dup (immtype:$imm))), 4721 (intvt ZPR:$Zs1), 4722 commuted_cc)), 4723 (cmp $Pg, $Zs1, immtype:$imm)>; 4724} 4725 4726multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4727 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4728 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4729 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4730 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4731 4732 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4733 !cast<Instruction>(NAME # _B)>; 4734 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4735 !cast<Instruction>(NAME # _H)>; 4736 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4737 !cast<Instruction>(NAME # _S)>; 4738 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4739 !cast<Instruction>(NAME # _D)>; 4740} 4741 4742 4743//===----------------------------------------------------------------------===// 4744// SVE Integer Compare - Unsigned Immediate Group 4745//===----------------------------------------------------------------------===// 4746 4747class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4748 ZPRRegOp zprty, Operand immtype> 4749: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4750 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4751 "", 4752 []>, Sched<[]> { 4753 bits<4> Pd; 4754 bits<3> Pg; 4755 bits<5> Zn; 4756 bits<7> imm7; 4757 let Inst{31-24} = 0b00100100; 4758 let Inst{23-22} = sz8_64; 4759 let Inst{21} = 1; 4760 let Inst{20-14} = imm7; 4761 let Inst{13} = opc{1}; 4762 let Inst{12-10} = Pg; 4763 let Inst{9-5} = Zn; 4764 let Inst{4} = opc{0}; 4765 let Inst{3-0} = Pd; 4766 4767 let Defs = [NZCV]; 4768 let ElementSize = pprty.ElementSize; 4769 let isPTestLike = 1; 4770} 4771 4772multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4773 CondCode commuted_cc> { 4774 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4775 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4776 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4777 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4778 4779 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4780 !cast<Instruction>(NAME # _B)>; 4781 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4782 !cast<Instruction>(NAME # _H)>; 4783 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4784 !cast<Instruction>(NAME # _S)>; 4785 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4786 !cast<Instruction>(NAME # _D)>; 4787} 4788 4789 4790//===----------------------------------------------------------------------===// 4791// SVE Integer Compare - Scalars Group 4792//===----------------------------------------------------------------------===// 4793 4794class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4795: I<(outs), (ins rt:$Rn, rt:$Rm), 4796 asm, "\t$Rn, $Rm", 4797 "", 4798 []>, Sched<[]> { 4799 bits<5> Rm; 4800 bits<5> Rn; 4801 let Inst{31-23} = 0b001001011; 4802 let Inst{22} = sz; 4803 let Inst{21} = 0b1; 4804 let Inst{20-16} = Rm; 4805 let Inst{15-10} = 0b001000; 4806 let Inst{9-5} = Rn; 4807 let Inst{4} = opc; 4808 let Inst{3-0} = 0b0000; 4809 4810 let Defs = [NZCV]; 4811} 4812 4813class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4814 RegisterClass gprty, PPRRegOp pprty> 4815: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4816 asm, "\t$Pd, $Rn, $Rm", 4817 "", []>, Sched<[]> { 4818 bits<4> Pd; 4819 bits<5> Rm; 4820 bits<5> Rn; 4821 let Inst{31-24} = 0b00100101; 4822 let Inst{23-22} = sz8_64; 4823 let Inst{21} = 0b1; 4824 let Inst{20-16} = Rm; 4825 let Inst{15-13} = 0b000; 4826 let Inst{12-10} = opc{3-1}; 4827 let Inst{9-5} = Rn; 4828 let Inst{4} = opc{0}; 4829 let Inst{3-0} = Pd; 4830 4831 let Defs = [NZCV]; 4832 let ElementSize = pprty.ElementSize; 4833 let isWhile = 1; 4834} 4835 4836multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4837 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 4838 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 4839 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 4840 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 4841 4842 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4843 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4844 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4845 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 4846} 4847 4848multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 4849 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 4850 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 4851 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 4852 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 4853 4854 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 4855 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 4856 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 4857 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 4858} 4859 4860class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 4861 PPRRegOp pprty> 4862: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 4863 asm, "\t$Pd, $Rn, $Rm", 4864 "", []>, Sched<[]> { 4865 bits<4> Pd; 4866 bits<5> Rm; 4867 bits<5> Rn; 4868 let Inst{31-24} = 0b00100101; 4869 let Inst{23-22} = sz8_64; 4870 let Inst{21} = 0b1; 4871 let Inst{20-16} = Rm; 4872 let Inst{15-10} = 0b001100; 4873 let Inst{9-5} = Rn; 4874 let Inst{4} = rw; 4875 let Inst{3-0} = Pd; 4876 4877 let Defs = [NZCV]; 4878 let ElementSize = pprty.ElementSize; 4879 let isWhile = 1; 4880} 4881 4882multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 4883 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 4884 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 4885 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 4886 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 4887 4888 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 4889 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 4890 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 4891 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 4892} 4893 4894//===----------------------------------------------------------------------===// 4895// SVE Floating Point Fast Reduction Group 4896//===----------------------------------------------------------------------===// 4897 4898class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 4899 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4900: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 4901 asm, "\t$Vd, $Pg, $Zn", 4902 "", 4903 []>, Sched<[]> { 4904 bits<5> Zn; 4905 bits<5> Vd; 4906 bits<3> Pg; 4907 let Inst{31-24} = 0b01100101; 4908 let Inst{23-22} = sz; 4909 let Inst{21-19} = 0b000; 4910 let Inst{18-16} = opc; 4911 let Inst{15-13} = 0b001; 4912 let Inst{12-10} = Pg; 4913 let Inst{9-5} = Zn; 4914 let Inst{4-0} = Vd; 4915} 4916 4917multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 4918 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 4919 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 4920 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 4921 4922 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4923 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4924 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4925 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4926 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4927 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4928} 4929 4930//===----------------------------------------------------------------------===// 4931// SVE Floating Point Accumulating Reduction Group 4932//===----------------------------------------------------------------------===// 4933 4934class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 4935 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4936: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 4937 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 4938 "", 4939 []>, 4940 Sched<[]> { 4941 bits<3> Pg; 4942 bits<5> Vdn; 4943 bits<5> Zm; 4944 let Inst{31-24} = 0b01100101; 4945 let Inst{23-22} = sz; 4946 let Inst{21-19} = 0b011; 4947 let Inst{18-16} = opc; 4948 let Inst{15-13} = 0b001; 4949 let Inst{12-10} = Pg; 4950 let Inst{9-5} = Zm; 4951 let Inst{4-0} = Vdn; 4952 4953 let Constraints = "$Vdn = $_Vdn"; 4954} 4955 4956multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 4957 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 4958 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 4959 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 4960 4961 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 4962 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 4963 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4964 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 4965 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4966 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4967} 4968 4969//===----------------------------------------------------------------------===// 4970// SVE Floating Point Compare - Vectors Group 4971//===----------------------------------------------------------------------===// 4972 4973class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4974 ZPRRegOp zprty> 4975: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 4976 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4977 "", 4978 []>, Sched<[]> { 4979 bits<4> Pd; 4980 bits<3> Pg; 4981 bits<5> Zm; 4982 bits<5> Zn; 4983 let Inst{31-24} = 0b01100101; 4984 let Inst{23-22} = sz; 4985 let Inst{21} = 0b0; 4986 let Inst{20-16} = Zm; 4987 let Inst{15} = opc{2}; 4988 let Inst{14} = 0b1; 4989 let Inst{13} = opc{1}; 4990 let Inst{12-10} = Pg; 4991 let Inst{9-5} = Zn; 4992 let Inst{4} = opc{0}; 4993 let Inst{3-0} = Pd; 4994} 4995 4996multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 4997 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4998 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4999 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5000 5001 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5002 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5003 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5004} 5005 5006multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, 5007 CondCode cc1, CondCode cc2, 5008 CondCode invcc1, CondCode invcc2> { 5009 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5010 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5011 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5012 5013 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5014 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5015 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5016 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5017 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5018 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5019 5020 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5021 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5022 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5023 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5024 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5025 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5026} 5027 5028//===----------------------------------------------------------------------===// 5029// SVE Floating Point Compare - with Zero Group 5030//===----------------------------------------------------------------------===// 5031 5032class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5033 ZPRRegOp zprty> 5034: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 5035 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 5036 "", 5037 []>, Sched<[]> { 5038 bits<4> Pd; 5039 bits<3> Pg; 5040 bits<5> Zn; 5041 let Inst{31-24} = 0b01100101; 5042 let Inst{23-22} = sz; 5043 let Inst{21-18} = 0b0100; 5044 let Inst{17-16} = opc{2-1}; 5045 let Inst{15-13} = 0b001; 5046 let Inst{12-10} = Pg; 5047 let Inst{9-5} = Zn; 5048 let Inst{4} = opc{0}; 5049 let Inst{3-0} = Pd; 5050} 5051 5052multiclass sve_fp_2op_p_pd<bits<3> opc, string asm, 5053 CondCode cc1, CondCode cc2, 5054 CondCode invcc1, CondCode invcc2> { 5055 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5056 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5057 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5058 5059 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5060 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5061 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5062 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5063 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5064 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5065 5066 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5067 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5068 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5069 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5070 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5071 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5072} 5073 5074 5075//===----------------------------------------------------------------------===// 5076//SVE Index Generation Group 5077//===----------------------------------------------------------------------===// 5078 5079def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>; 5080def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>; 5081def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>; 5082def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>; 5083def i64imm_32bit_tgt : TImmLeaf<i64, [{ 5084 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); 5085}]>; 5086 5087class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5088 Operand imm_ty> 5089: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 5090 asm, "\t$Zd, $imm5, $imm5b", 5091 "", []>, Sched<[]> { 5092 bits<5> Zd; 5093 bits<5> imm5; 5094 bits<5> imm5b; 5095 let Inst{31-24} = 0b00000100; 5096 let Inst{23-22} = sz8_64; 5097 let Inst{21} = 0b1; 5098 let Inst{20-16} = imm5b; 5099 let Inst{15-10} = 0b010000; 5100 let Inst{9-5} = imm5; 5101 let Inst{4-0} = Zd; 5102} 5103 5104multiclass sve_int_index_ii<string asm> { 5105 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 5106 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 5107 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 5108 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 5109 5110 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)), 5111 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5112 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)), 5113 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5114 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)), 5115 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>; 5116 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)), 5117 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>; 5118 5119 // add(step_vector(step), dup(X)) -> index(X, step). 5120 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))), 5121 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5122 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))), 5123 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5124 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))), 5125 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 5126 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5127 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 5128} 5129 5130class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5131 RegisterClass srcRegType, Operand imm_ty> 5132: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 5133 asm, "\t$Zd, $imm5, $Rm", 5134 "", []>, Sched<[]> { 5135 bits<5> Rm; 5136 bits<5> Zd; 5137 bits<5> imm5; 5138 let Inst{31-24} = 0b00000100; 5139 let Inst{23-22} = sz8_64; 5140 let Inst{21} = 0b1; 5141 let Inst{20-16} = Rm; 5142 let Inst{15-10} = 0b010010; 5143 let Inst{9-5} = imm5; 5144 let Inst{4-0} = Zd; 5145} 5146 5147multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> { 5148 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 5149 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 5150 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 5151 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 5152 5153 def : Pat<(nxv16i8 (step_vector i8:$imm)), 5154 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5155 def : Pat<(nxv8i16 (step_vector i16:$imm)), 5156 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5157 def : Pat<(nxv4i32 (step_vector i32:$imm)), 5158 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>; 5159 def : Pat<(nxv2i64 (step_vector i64:$imm)), 5160 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>; 5161 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)), 5162 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5163 5164 // add(step_vector(step), dup(X)) -> index(X, step). 5165 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))), 5166 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5167 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))), 5168 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5169 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))), 5170 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>; 5171 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5172 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>; 5173 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5174 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5175 5176 // mul(step_vector(1), dup(Y)) -> index(0, Y). 5177 def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), 5178 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>; 5179 def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), 5180 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>; 5181 def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), 5182 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>; 5183 def : Pat<(mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))), 5184 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>; 5185 5186 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5187 def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))), 5188 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 5189 def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))), 5190 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 5191 def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))), 5192 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 5193 def : Pat<(add (muloneuseop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5194 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 5195} 5196 5197class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5198 RegisterClass srcRegType, Operand imm_ty> 5199: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 5200 asm, "\t$Zd, $Rn, $imm5", 5201 "", []>, Sched<[]> { 5202 bits<5> Rn; 5203 bits<5> Zd; 5204 bits<5> imm5; 5205 let Inst{31-24} = 0b00000100; 5206 let Inst{23-22} = sz8_64; 5207 let Inst{21} = 0b1; 5208 let Inst{20-16} = imm5; 5209 let Inst{15-10} = 0b010001; 5210 let Inst{9-5} = Rn; 5211 let Inst{4-0} = Zd; 5212} 5213 5214multiclass sve_int_index_ri<string asm> { 5215 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 5216 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 5217 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 5218 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 5219 5220 // add(step_vector(step), dup(X)) -> index(X, step). 5221 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), 5222 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5223 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), 5224 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5225 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), 5226 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 5227 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))), 5228 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 5229} 5230 5231class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5232 RegisterClass srcRegType> 5233: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 5234 asm, "\t$Zd, $Rn, $Rm", 5235 "", []>, Sched<[]> { 5236 bits<5> Zd; 5237 bits<5> Rm; 5238 bits<5> Rn; 5239 let Inst{31-24} = 0b00000100; 5240 let Inst{23-22} = sz8_64; 5241 let Inst{21} = 0b1; 5242 let Inst{20-16} = Rm; 5243 let Inst{15-10} = 0b010011; 5244 let Inst{9-5} = Rn; 5245 let Inst{4-0} = Zd; 5246} 5247 5248multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> { 5249 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 5250 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 5251 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 5252 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 5253 5254 // add(step_vector(step), dup(X)) -> index(X, step). 5255 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))), 5256 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5257 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (AArch64dup(i32 GPR32:$Rn)))), 5258 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5259 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (AArch64dup(i32 GPR32:$Rn)))), 5260 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>; 5261 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))), 5262 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>; 5263 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))), 5264 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5265 5266 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5267 def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))), 5268 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>; 5269 def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),(nxv8i16 (AArch64dup(i32 GPR32:$Rn)))), 5270 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>; 5271 def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))),(nxv4i32 (AArch64dup(i32 GPR32:$Rn)))), 5272 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>; 5273 def : Pat<(add (mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))),(nxv2i64 (AArch64dup(i64 GPR64:$Rn)))), 5274 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>; 5275} 5276 5277//===----------------------------------------------------------------------===// 5278// SVE Bitwise Shift - Predicated Group 5279//===----------------------------------------------------------------------===// 5280 5281class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 5282 ZPRRegOp zprty, Operand immtype> 5283: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 5284 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 5285 "", 5286 []>, Sched<[]> { 5287 bits<3> Pg; 5288 bits<5> Zdn; 5289 bits<6> imm; 5290 let Inst{31-24} = 0b00000100; 5291 let Inst{23-22} = tsz8_64{3-2}; 5292 let Inst{21-20} = 0b00; 5293 let Inst{19-16} = opc; 5294 let Inst{15-13} = 0b100; 5295 let Inst{12-10} = Pg; 5296 let Inst{9-8} = tsz8_64{1-0}; 5297 let Inst{7-5} = imm{2-0}; // imm3 5298 let Inst{4-0} = Zdn; 5299 5300 let Constraints = "$Zdn = $_Zdn"; 5301 let DestructiveInstType = DestructiveBinaryImm; 5302 let ElementSize = zprty.ElementSize; 5303} 5304 5305multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 5306 SDPatternOperator op = null_frag> { 5307 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5308 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5309 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5310 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5311 let Inst{8} = imm{3}; 5312 } 5313 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5314 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5315 let Inst{9-8} = imm{4-3}; 5316 } 5317 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5318 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5319 let Inst{22} = imm{5}; 5320 let Inst{9-8} = imm{4-3}; 5321 } 5322 5323 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 5324 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 5325 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 5326 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 5327} 5328 5329// As above but shift amount takes the form of a "vector immediate". 5330multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 5331 string Ps, SDPatternOperator op> 5332: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 5333 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5334 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5335 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5336 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5337} 5338 5339multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 5340 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 5341 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 5342 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 5343 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 5344 5345 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 5346 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 5347 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 5348 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 5349} 5350 5351multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 5352 SDPatternOperator op = null_frag> { 5353 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5354 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5355 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5356 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5357 let Inst{8} = imm{3}; 5358 } 5359 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5360 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5361 let Inst{9-8} = imm{4-3}; 5362 } 5363 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5364 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5365 let Inst{22} = imm{5}; 5366 let Inst{9-8} = imm{4-3}; 5367 } 5368 5369 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 5370 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 5371 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 5372 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 5373} 5374 5375// As above but shift amount takes the form of a "vector immediate". 5376multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 5377 string Ps, SDPatternOperator op> 5378: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 5379 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5380 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5381 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5382 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5383} 5384 5385multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 5386 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 5387 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 5388 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 5389 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 5390 5391 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 5392 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 5393 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 5394 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 5395} 5396 5397class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 5398 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 5399: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 5400 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 5401 "", 5402 []>, Sched<[]> { 5403 bits<3> Pg; 5404 bits<5> Zdn; 5405 bits<5> Zm; 5406 let Inst{31-24} = 0b00000100; 5407 let Inst{23-22} = sz8_64; 5408 let Inst{21-20} = 0b01; 5409 let Inst{19} = wide; 5410 let Inst{18-16} = opc; 5411 let Inst{15-13} = 0b100; 5412 let Inst{12-10} = Pg; 5413 let Inst{9-5} = Zm; 5414 let Inst{4-0} = Zdn; 5415 5416 let Constraints = "$Zdn = $_Zdn"; 5417 let DestructiveInstType = DestructiveOther; 5418 let ElementSize = zprty.ElementSize; 5419} 5420 5421multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 5422 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 5423 let DestructiveInstType = DestructiveBinaryCommWithRev in { 5424 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 5425 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 5426 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 5427 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 5428 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 5429 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 5430 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 5431 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 5432 } 5433 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5434 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5435 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5436 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5437} 5438 5439multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5440 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5441 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5442 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5443 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5444 5445 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5446 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5447 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5448 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5449} 5450 5451multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5452 SDPatternOperator op> { 5453 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5454 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5455 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5456 5457 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5458 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5459 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5460} 5461 5462//===----------------------------------------------------------------------===// 5463// SVE Shift - Unpredicated Group 5464//===----------------------------------------------------------------------===// 5465 5466class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5467 ZPRRegOp zprty> 5468: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5469 asm, "\t$Zd, $Zn, $Zm", 5470 "", 5471 []>, Sched<[]> { 5472 bits<5> Zd; 5473 bits<5> Zm; 5474 bits<5> Zn; 5475 let Inst{31-24} = 0b00000100; 5476 let Inst{23-22} = sz8_64; 5477 let Inst{21} = 0b1; 5478 let Inst{20-16} = Zm; 5479 let Inst{15-12} = 0b1000; 5480 let Inst{11-10} = opc; 5481 let Inst{9-5} = Zn; 5482 let Inst{4-0} = Zd; 5483} 5484 5485multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> { 5486 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5487 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5488 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5489 5490 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5491 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5492 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5493} 5494 5495class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5496 ZPRRegOp zprty, Operand immtype> 5497: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5498 asm, "\t$Zd, $Zn, $imm", 5499 "", 5500 []>, Sched<[]> { 5501 bits<5> Zd; 5502 bits<5> Zn; 5503 bits<6> imm; 5504 let Inst{31-24} = 0b00000100; 5505 let Inst{23-22} = tsz8_64{3-2}; 5506 let Inst{21} = 0b1; 5507 let Inst{20-19} = tsz8_64{1-0}; 5508 let Inst{18-16} = imm{2-0}; // imm3 5509 let Inst{15-12} = 0b1001; 5510 let Inst{11-10} = opc; 5511 let Inst{9-5} = Zn; 5512 let Inst{4-0} = Zd; 5513} 5514 5515multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5516 SDPatternOperator op> { 5517 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5518 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5519 let Inst{19} = imm{3}; 5520 } 5521 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5522 let Inst{20-19} = imm{4-3}; 5523 } 5524 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5525 let Inst{22} = imm{5}; 5526 let Inst{20-19} = imm{4-3}; 5527 } 5528 5529 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5530 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5531 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5532 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5533} 5534 5535multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5536 SDPatternOperator op> { 5537 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5538 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5539 let Inst{19} = imm{3}; 5540 } 5541 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5542 let Inst{20-19} = imm{4-3}; 5543 } 5544 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5545 let Inst{22} = imm{5}; 5546 let Inst{20-19} = imm{4-3}; 5547 } 5548 5549 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5550 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5551 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5552 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5553} 5554 5555//===----------------------------------------------------------------------===// 5556// SVE Memory - Store Group 5557//===----------------------------------------------------------------------===// 5558 5559class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5560 RegisterOperand VecList> 5561: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5562 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5563 "", 5564 []>, Sched<[]> { 5565 bits<3> Pg; 5566 bits<5> Rn; 5567 bits<5> Zt; 5568 bits<4> imm4; 5569 let Inst{31-25} = 0b1110010; 5570 let Inst{24-23} = msz; 5571 let Inst{22-21} = esz; 5572 let Inst{20} = 0; 5573 let Inst{19-16} = imm4; 5574 let Inst{15-13} = 0b111; 5575 let Inst{12-10} = Pg; 5576 let Inst{9-5} = Rn; 5577 let Inst{4-0} = Zt; 5578 5579 let mayStore = 1; 5580} 5581 5582multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5583 RegisterOperand listty, ZPRRegOp zprty> 5584{ 5585 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5586 5587 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5588 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5589 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5590 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5591 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5592 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5593} 5594 5595class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5596 string asm, Operand immtype> 5597: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5598 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5599 "", 5600 []>, Sched<[]> { 5601 bits<3> Pg; 5602 bits<5> Rn; 5603 bits<5> Zt; 5604 bits<4> imm4; 5605 let Inst{31-25} = 0b1110010; 5606 let Inst{24-23} = sz; 5607 let Inst{22-21} = nregs; 5608 let Inst{20} = 1; 5609 let Inst{19-16} = imm4; 5610 let Inst{15-13} = 0b111; 5611 let Inst{12-10} = Pg; 5612 let Inst{9-5} = Rn; 5613 let Inst{4-0} = Zt; 5614 5615 let mayStore = 1; 5616} 5617 5618multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5619 string asm, Operand immtype> { 5620 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5621 5622 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5623 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5624} 5625 5626class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5627 string asm, RegisterOperand gprty> 5628: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5629 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5630 "", 5631 []>, Sched<[]> { 5632 bits<3> Pg; 5633 bits<5> Rm; 5634 bits<5> Rn; 5635 bits<5> Zt; 5636 let Inst{31-25} = 0b1110010; 5637 let Inst{24-23} = sz; 5638 let Inst{22-21} = nregs; 5639 let Inst{20-16} = Rm; 5640 let Inst{15-13} = 0b011; 5641 let Inst{12-10} = Pg; 5642 let Inst{9-5} = Rn; 5643 let Inst{4-0} = Zt; 5644 5645 let mayStore = 1; 5646} 5647 5648class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5649 RegisterOperand listty, RegisterOperand gprty> 5650: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5651 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5652 "", 5653 []>, Sched<[]> { 5654 bits<3> Pg; 5655 bits<5> Rm; 5656 bits<5> Rn; 5657 bits<5> Zt; 5658 let Inst{31-25} = 0b1110010; 5659 let Inst{24-21} = dtype; 5660 let Inst{20-16} = Rm; 5661 let Inst{15-13} = 0b010; 5662 let Inst{12-10} = Pg; 5663 let Inst{9-5} = Rn; 5664 let Inst{4-0} = Zt; 5665 5666 let mayStore = 1; 5667} 5668 5669multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5670 RegisterOperand listty, ZPRRegOp zprty, 5671 RegisterOperand gprty> { 5672 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5673 5674 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5675 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5676} 5677 5678class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5679: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5680 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5681 "", 5682 []>, Sched<[]> { 5683 bits<3> Pg; 5684 bits<5> Rn; 5685 bits<5> Zt; 5686 bits<4> imm4; 5687 let Inst{31-25} = 0b1110010; 5688 let Inst{24-23} = msz; 5689 let Inst{22-20} = 0b001; 5690 let Inst{19-16} = imm4; 5691 let Inst{15-13} = 0b111; 5692 let Inst{12-10} = Pg; 5693 let Inst{9-5} = Rn; 5694 let Inst{4-0} = Zt; 5695 5696 let mayStore = 1; 5697} 5698 5699multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5700 ZPRRegOp zprty> { 5701 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5702 5703 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5704 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5705 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5706 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5707 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5708 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5709} 5710 5711class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5712 RegisterOperand gprty> 5713: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5714 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5715 "", 5716 []>, Sched<[]> { 5717 bits<3> Pg; 5718 bits<5> Rm; 5719 bits<5> Rn; 5720 bits<5> Zt; 5721 let Inst{31-25} = 0b1110010; 5722 let Inst{24-23} = msz; 5723 let Inst{22-21} = 0b00; 5724 let Inst{20-16} = Rm; 5725 let Inst{15-13} = 0b011; 5726 let Inst{12-10} = Pg; 5727 let Inst{9-5} = Rn; 5728 let Inst{4-0} = Zt; 5729 5730 let mayStore = 1; 5731} 5732 5733multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5734 ZPRRegOp zprty, RegisterOperand gprty> { 5735 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5736 5737 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5738 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5739} 5740 5741class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5742 RegisterOperand listty, ZPRRegOp zprty> 5743: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5744 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5745 "", 5746 []>, Sched<[]> { 5747 bits<3> Pg; 5748 bits<5> Rm; 5749 bits<5> Zn; 5750 bits<5> Zt; 5751 let Inst{31-25} = 0b1110010; 5752 let Inst{24-22} = opc; 5753 let Inst{21} = 0b0; 5754 let Inst{20-16} = Rm; 5755 let Inst{15-13} = 0b001; 5756 let Inst{12-10} = Pg; 5757 let Inst{9-5} = Zn; 5758 let Inst{4-0} = Zt; 5759 5760 let mayStore = 1; 5761} 5762 5763multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5764 SDPatternOperator op, 5765 ValueType vt> { 5766 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5767 5768 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5769 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5770 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5771 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5772 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5773 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5774 5775 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5776 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5777} 5778 5779multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5780 SDPatternOperator op, 5781 ValueType vt> { 5782 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5783 5784 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5785 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5786 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5787 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5788 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5789 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5790 5791 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5792 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5793} 5794 5795class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5796 RegisterOperand VecList, RegisterOperand zprext> 5797: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5798 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5799 "", 5800 []>, Sched<[]> { 5801 bits<3> Pg; 5802 bits<5> Rn; 5803 bits<5> Zm; 5804 bits<5> Zt; 5805 let Inst{31-25} = 0b1110010; 5806 let Inst{24-22} = opc; 5807 let Inst{21} = scaled; 5808 let Inst{20-16} = Zm; 5809 let Inst{15} = 0b1; 5810 let Inst{14} = xs; 5811 let Inst{13} = 0; 5812 let Inst{12-10} = Pg; 5813 let Inst{9-5} = Rn; 5814 let Inst{4-0} = Zt; 5815 5816 let mayStore = 1; 5817} 5818 5819multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5820 SDPatternOperator sxtw_op, 5821 SDPatternOperator uxtw_op, 5822 RegisterOperand sxtw_opnd, 5823 RegisterOperand uxtw_opnd, 5824 ValueType vt > { 5825 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5826 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5827 5828 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5829 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5830 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5831 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5832 5833 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5834 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5835 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5836 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5837} 5838 5839multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5840 SDPatternOperator sxtw_op, 5841 SDPatternOperator uxtw_op, 5842 RegisterOperand sxtw_opnd, 5843 RegisterOperand uxtw_opnd, 5844 ValueType vt > { 5845 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 5846 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 5847 5848 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5849 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5850 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5851 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5852 5853 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5854 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5855 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5856 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5857} 5858 5859multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 5860 SDPatternOperator sxtw_op, 5861 SDPatternOperator uxtw_op, 5862 RegisterOperand sxtw_opnd, 5863 RegisterOperand uxtw_opnd, 5864 ValueType vt> { 5865 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 5866 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 5867 5868 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5869 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5870 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5871 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5872 5873 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5874 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5875 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5876 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5877} 5878 5879multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 5880 SDPatternOperator sxtw_op, 5881 SDPatternOperator uxtw_op, 5882 RegisterOperand sxtw_opnd, 5883 RegisterOperand uxtw_opnd, 5884 ValueType vt> { 5885 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 5886 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 5887 5888 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5889 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5890 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5891 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5892 5893 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5894 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5895 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5896 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5897} 5898 5899class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 5900 RegisterOperand zprext> 5901: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5902 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5903 "", 5904 []>, Sched<[]> { 5905 bits<3> Pg; 5906 bits<5> Rn; 5907 bits<5> Zm; 5908 bits<5> Zt; 5909 let Inst{31-25} = 0b1110010; 5910 let Inst{24-23} = msz; 5911 let Inst{22} = 0b0; 5912 let Inst{21} = scaled; 5913 let Inst{20-16} = Zm; 5914 let Inst{15-13} = 0b101; 5915 let Inst{12-10} = Pg; 5916 let Inst{9-5} = Rn; 5917 let Inst{4-0} = Zt; 5918 5919 let mayStore = 1; 5920} 5921 5922multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 5923 SDPatternOperator op, 5924 RegisterOperand zprext, 5925 ValueType vt> { 5926 def _SCALED_REAL : sve_mem_sst_sv2<msz, 1, asm, zprext>; 5927 5928 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5929 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 5930 5931 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 5932 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 5933} 5934 5935multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 5936 SDPatternOperator op, 5937 ValueType vt> { 5938 def _REAL : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 5939 5940 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5941 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 5942 5943 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5944 (!cast<Instruction>(NAME # _REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5945} 5946 5947class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 5948 RegisterOperand VecList, Operand imm_ty> 5949: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 5950 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 5951 "", 5952 []>, Sched<[]> { 5953 bits<3> Pg; 5954 bits<5> imm5; 5955 bits<5> Zn; 5956 bits<5> Zt; 5957 let Inst{31-25} = 0b1110010; 5958 let Inst{24-23} = opc{2-1}; 5959 let Inst{22} = 0b1; 5960 let Inst{21} = opc{0}; 5961 let Inst{20-16} = imm5; 5962 let Inst{15-13} = 0b101; 5963 let Inst{12-10} = Pg; 5964 let Inst{9-5} = Zn; 5965 let Inst{4-0} = Zt; 5966 5967 let mayStore = 1; 5968} 5969 5970multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 5971 Operand imm_ty, 5972 SDPatternOperator op, 5973 ValueType vt> { 5974 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 5975 5976 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5977 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 5978 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5979 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 5980 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5981 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 5982 5983 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 5984 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5985} 5986 5987multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 5988 Operand imm_ty, 5989 SDPatternOperator op, 5990 ValueType vt> { 5991 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 5992 5993 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5994 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 5995 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5996 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 5997 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5998 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 5999 6000 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 6001 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6002} 6003 6004class sve_mem_z_spill<string asm> 6005: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 6006 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6007 "", 6008 []>, Sched<[]> { 6009 bits<5> Rn; 6010 bits<5> Zt; 6011 bits<9> imm9; 6012 let Inst{31-22} = 0b1110010110; 6013 let Inst{21-16} = imm9{8-3}; 6014 let Inst{15-13} = 0b010; 6015 let Inst{12-10} = imm9{2-0}; 6016 let Inst{9-5} = Rn; 6017 let Inst{4-0} = Zt; 6018 6019 let mayStore = 1; 6020} 6021 6022multiclass sve_mem_z_spill<string asm> { 6023 def NAME : sve_mem_z_spill<asm>; 6024 6025 def : InstAlias<asm # "\t$Zt, [$Rn]", 6026 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6027} 6028 6029class sve_mem_p_spill<string asm> 6030: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 6031 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6032 "", 6033 []>, Sched<[]> { 6034 bits<4> Pt; 6035 bits<5> Rn; 6036 bits<9> imm9; 6037 let Inst{31-22} = 0b1110010110; 6038 let Inst{21-16} = imm9{8-3}; 6039 let Inst{15-13} = 0b000; 6040 let Inst{12-10} = imm9{2-0}; 6041 let Inst{9-5} = Rn; 6042 let Inst{4} = 0b0; 6043 let Inst{3-0} = Pt; 6044 6045 let mayStore = 1; 6046} 6047 6048multiclass sve_mem_p_spill<string asm> { 6049 def NAME : sve_mem_p_spill<asm>; 6050 6051 def : InstAlias<asm # "\t$Pt, [$Rn]", 6052 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6053} 6054 6055//===----------------------------------------------------------------------===// 6056// SVE Permute - Predicates Group 6057//===----------------------------------------------------------------------===// 6058 6059class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 6060 PPRRegOp pprty> 6061: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 6062 asm, "\t$Pd, $Pn, $Pm", 6063 "", []>, Sched<[]> { 6064 bits<4> Pd; 6065 bits<4> Pm; 6066 bits<4> Pn; 6067 let Inst{31-24} = 0b00000101; 6068 let Inst{23-22} = sz8_64; 6069 let Inst{21-20} = 0b10; 6070 let Inst{19-16} = Pm; 6071 let Inst{15-13} = 0b010; 6072 let Inst{12-10} = opc; 6073 let Inst{9} = 0b0; 6074 let Inst{8-5} = Pn; 6075 let Inst{4} = 0b0; 6076 let Inst{3-0} = Pd; 6077} 6078 6079multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 6080 SDPatternOperator op> { 6081 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 6082 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 6083 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 6084 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 6085 6086 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 6087 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 6088 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 6089 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 6090} 6091 6092class sve_int_perm_punpk<bit opc, string asm> 6093: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 6094 asm, "\t$Pd, $Pn", 6095 "", 6096 []>, Sched<[]> { 6097 bits<4> Pd; 6098 bits<4> Pn; 6099 let Inst{31-17} = 0b000001010011000; 6100 let Inst{16} = opc; 6101 let Inst{15-9} = 0b0100000; 6102 let Inst{8-5} = Pn; 6103 let Inst{4} = 0b0; 6104 let Inst{3-0} = Pd; 6105} 6106 6107multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 6108 def NAME : sve_int_perm_punpk<opc, asm>; 6109 6110 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 6111 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 6112 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 6113} 6114 6115class sve_int_rdffr_pred<bit s, string asm> 6116: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 6117 asm, "\t$Pd, $Pg/z", 6118 "", 6119 []>, Sched<[]> { 6120 bits<4> Pd; 6121 bits<4> Pg; 6122 let Inst{31-23} = 0b001001010; 6123 let Inst{22} = s; 6124 let Inst{21-9} = 0b0110001111000; 6125 let Inst{8-5} = Pg; 6126 let Inst{4} = 0; 6127 let Inst{3-0} = Pd; 6128 6129 let Defs = !if(s, [NZCV], []); 6130 let Uses = [FFR]; 6131} 6132 6133multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 6134 def _REAL : sve_int_rdffr_pred<s, asm>; 6135 6136 // We need a layer of indirection because early machine code passes balk at 6137 // physical register (i.e. FFR) uses that have no previous definition. 6138 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6139 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 6140 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 6141 } 6142} 6143 6144class sve_int_rdffr_unpred<string asm> : I< 6145 (outs PPR8:$Pd), (ins), 6146 asm, "\t$Pd", 6147 "", 6148 []>, Sched<[]> { 6149 bits<4> Pd; 6150 let Inst{31-4} = 0b0010010100011001111100000000; 6151 let Inst{3-0} = Pd; 6152 6153 let Uses = [FFR]; 6154} 6155 6156multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 6157 def _REAL : sve_int_rdffr_unpred<asm>; 6158 6159 // We need a layer of indirection because early machine code passes balk at 6160 // physical register (i.e. FFR) uses that have no previous definition. 6161 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6162 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 6163 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 6164 } 6165} 6166 6167class sve_int_wrffr<string asm, SDPatternOperator op> 6168: I<(outs), (ins PPR8:$Pn), 6169 asm, "\t$Pn", 6170 "", 6171 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 6172 bits<4> Pn; 6173 let Inst{31-9} = 0b00100101001010001001000; 6174 let Inst{8-5} = Pn; 6175 let Inst{4-0} = 0b00000; 6176 6177 let hasSideEffects = 1; 6178 let Defs = [FFR]; 6179} 6180 6181class sve_int_setffr<string asm, SDPatternOperator op> 6182: I<(outs), (ins), 6183 asm, "", 6184 "", 6185 [(op)]>, Sched<[]> { 6186 let Inst{31-0} = 0b00100101001011001001000000000000; 6187 6188 let hasSideEffects = 1; 6189 let Defs = [FFR]; 6190} 6191 6192//===----------------------------------------------------------------------===// 6193// SVE Permute Vector - Predicated Group 6194//===----------------------------------------------------------------------===// 6195 6196class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 6197 ZPRRegOp zprty, RegisterClass rt> 6198: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 6199 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 6200 "", 6201 []>, Sched<[]> { 6202 bits<3> Pg; 6203 bits<5> Rdn; 6204 bits<5> Zm; 6205 let Inst{31-24} = 0b00000101; 6206 let Inst{23-22} = sz8_64; 6207 let Inst{21-17} = 0b11000; 6208 let Inst{16} = ab; 6209 let Inst{15-13} = 0b101; 6210 let Inst{12-10} = Pg; 6211 let Inst{9-5} = Zm; 6212 let Inst{4-0} = Rdn; 6213 6214 let Constraints = "$Rdn = $_Rdn"; 6215} 6216 6217multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 6218 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 6219 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 6220 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 6221 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 6222 6223 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 6224 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 6225 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6226 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6227} 6228 6229class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 6230 ZPRRegOp zprty, RegisterClass rt> 6231: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 6232 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 6233 "", 6234 []>, Sched<[]> { 6235 bits<3> Pg; 6236 bits<5> Vdn; 6237 bits<5> Zm; 6238 let Inst{31-24} = 0b00000101; 6239 let Inst{23-22} = sz8_64; 6240 let Inst{21-17} = 0b10101; 6241 let Inst{16} = ab; 6242 let Inst{15-13} = 0b100; 6243 let Inst{12-10} = Pg; 6244 let Inst{9-5} = Zm; 6245 let Inst{4-0} = Vdn; 6246 6247 let Constraints = "$Vdn = $_Vdn"; 6248} 6249 6250multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 6251 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 6252 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 6253 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 6254 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 6255 6256 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6257 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6258 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6259 6260 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6261} 6262 6263class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 6264 ZPRRegOp zprty> 6265: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6266 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6267 "", 6268 []>, Sched<[]> { 6269 bits<3> Pg; 6270 bits<5> Zdn; 6271 bits<5> Zm; 6272 let Inst{31-24} = 0b00000101; 6273 let Inst{23-22} = sz8_64; 6274 let Inst{21-17} = 0b10100; 6275 let Inst{16} = ab; 6276 let Inst{15-13} = 0b100; 6277 let Inst{12-10} = Pg; 6278 let Inst{9-5} = Zm; 6279 let Inst{4-0} = Zdn; 6280 6281 let Constraints = "$Zdn = $_Zdn"; 6282 let DestructiveInstType = DestructiveOther; 6283 let ElementSize = ElementSizeNone; 6284} 6285 6286multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 6287 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 6288 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 6289 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 6290 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 6291 6292 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6293 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6294 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6295 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6296 6297 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6298 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6299 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6300 6301 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6302} 6303 6304class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 6305 ZPRRegOp zprty, RegisterClass resultRegType> 6306: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 6307 asm, "\t$Rd, $Pg, $Zn", 6308 "", 6309 []>, Sched<[]> { 6310 bits<3> Pg; 6311 bits<5> Rd; 6312 bits<5> Zn; 6313 let Inst{31-24} = 0b00000101; 6314 let Inst{23-22} = sz8_64; 6315 let Inst{21-17} = 0b10000; 6316 let Inst{16} = ab; 6317 let Inst{15-13} = 0b101; 6318 let Inst{12-10} = Pg; 6319 let Inst{9-5} = Zn; 6320 let Inst{4-0} = Rd; 6321} 6322 6323multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 6324 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 6325 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 6326 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 6327 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 6328 6329 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6330 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6331 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6332 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6333} 6334 6335class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 6336 ZPRRegOp zprty, RegisterClass dstRegtype> 6337: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 6338 asm, "\t$Vd, $Pg, $Zn", 6339 "", 6340 []>, Sched<[]> { 6341 bits<3> Pg; 6342 bits<5> Vd; 6343 bits<5> Zn; 6344 let Inst{31-24} = 0b00000101; 6345 let Inst{23-22} = sz8_64; 6346 let Inst{21-17} = 0b10001; 6347 let Inst{16} = ab; 6348 let Inst{15-13} = 0b100; 6349 let Inst{12-10} = Pg; 6350 let Inst{9-5} = Zn; 6351 let Inst{4-0} = Vd; 6352} 6353 6354multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 6355 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 6356 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 6357 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 6358 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 6359 6360 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 6361 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6362 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 6363 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6364 6365 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6366} 6367 6368class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 6369: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6370 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6371 "", 6372 []>, Sched<[]> { 6373 bits<3> Pg; 6374 bits<5> Zdn; 6375 bits<5> Zm; 6376 let Inst{31-24} = 0b00000101; 6377 let Inst{23-22} = sz8_64; 6378 let Inst{21-13} = 0b101100100; 6379 let Inst{12-10} = Pg; 6380 let Inst{9-5} = Zm; 6381 let Inst{4-0} = Zdn; 6382 6383 let Constraints = "$Zdn = $_Zdn"; 6384 let DestructiveInstType = DestructiveOther; 6385 let ElementSize = ElementSizeNone; 6386} 6387 6388multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 6389 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 6390 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 6391 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 6392 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 6393 6394 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6395 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6396 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6397 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6398 6399 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6400 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6401 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6402 6403 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6404} 6405 6406class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 6407 ZPRRegOp zprty, RegisterOperand VecList> 6408: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 6409 asm, "\t$Zd, $Pg, $Zn", 6410 "", 6411 []>, Sched<[]> { 6412 bits<3> Pg; 6413 bits<5> Zn; 6414 bits<5> Zd; 6415 let Inst{31-24} = 0b00000101; 6416 let Inst{23-22} = sz8_64; 6417 let Inst{21-13} = 0b101101100; 6418 let Inst{12-10} = Pg; 6419 let Inst{9-5} = Zn; 6420 let Inst{4-0} = Zd; 6421} 6422 6423multiclass sve2_int_perm_splice_cons<string asm> { 6424 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 6425 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 6426 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 6427 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 6428} 6429 6430class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 6431 ZPRRegOp zprty> 6432: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 6433 asm, "\t$Zd, $Pg/m, $Zn", 6434 "", 6435 []>, Sched<[]> { 6436 bits<5> Zd; 6437 bits<3> Pg; 6438 bits<5> Zn; 6439 let Inst{31-24} = 0b00000101; 6440 let Inst{23-22} = sz8_64; 6441 let Inst{21-18} = 0b1001; 6442 let Inst{17-16} = opc; 6443 let Inst{15-13} = 0b100; 6444 let Inst{12-10} = Pg; 6445 let Inst{9-5} = Zn; 6446 let Inst{4-0} = Zd; 6447 6448 let Constraints = "$Zd = $_Zd"; 6449 let DestructiveInstType = DestructiveOther; 6450 let ElementSize = zprty.ElementSize; 6451} 6452 6453multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6454 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6455 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6456 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6457 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6458 6459 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6460 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6461 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6462 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6463} 6464 6465multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6466 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6467 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6468 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6469 6470 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6471 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6472 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6473} 6474 6475multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6476 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6477 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6478 6479 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6480 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6481} 6482 6483multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6484 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6485 6486 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6487} 6488 6489class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6490 RegisterClass srcRegType> 6491: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6492 asm, "\t$Zd, $Pg/m, $Rn", 6493 "", 6494 []>, Sched<[]> { 6495 bits<3> Pg; 6496 bits<5> Rn; 6497 bits<5> Zd; 6498 let Inst{31-24} = 0b00000101; 6499 let Inst{23-22} = sz8_64; 6500 let Inst{21-13} = 0b101000101; 6501 let Inst{12-10} = Pg; 6502 let Inst{9-5} = Rn; 6503 let Inst{4-0} = Zd; 6504 6505 let Constraints = "$Zd = $_Zd"; 6506 let DestructiveInstType = DestructiveOther; 6507 let ElementSize = zprty.ElementSize; 6508} 6509 6510multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6511 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6512 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6513 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6514 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6515 6516 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6517 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6518 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6519 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6520 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6521 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6522 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6523 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6524 6525 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6526 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6527 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6528 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6529 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6530 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6531 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6532 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6533} 6534 6535class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6536 RegisterClass srcRegtype> 6537: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6538 asm, "\t$Zd, $Pg/m, $Vn", 6539 "", 6540 []>, Sched<[]> { 6541 bits<3> Pg; 6542 bits<5> Vn; 6543 bits<5> Zd; 6544 let Inst{31-24} = 0b00000101; 6545 let Inst{23-22} = sz8_64; 6546 let Inst{21-13} = 0b100000100; 6547 let Inst{12-10} = Pg; 6548 let Inst{9-5} = Vn; 6549 let Inst{4-0} = Zd; 6550 6551 let Constraints = "$Zd = $_Zd"; 6552 let DestructiveInstType = DestructiveOther; 6553 let ElementSize = zprty.ElementSize; 6554} 6555 6556multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6557 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6558 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6559 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6560 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6561 6562 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6563 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6564 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6565 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6566 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6567 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6568 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6569 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6570 6571 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6572 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6573 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6574 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6575 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6576 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6577 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6578 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6579 6580 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6581 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6582} 6583 6584class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6585: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6586 asm, "\t$Zd, $Pg, $Zn", 6587 "", 6588 []>, Sched<[]> { 6589 bits<3> Pg; 6590 bits<5> Zd; 6591 bits<5> Zn; 6592 let Inst{31-23} = 0b000001011; 6593 let Inst{22} = sz; 6594 let Inst{21-13} = 0b100001100; 6595 let Inst{12-10} = Pg; 6596 let Inst{9-5} = Zn; 6597 let Inst{4-0} = Zd; 6598} 6599 6600multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6601 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6602 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6603 6604 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6605 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6606 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6607 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6608} 6609 6610//===----------------------------------------------------------------------===// 6611// SVE Memory - Contiguous Load Group 6612//===----------------------------------------------------------------------===// 6613 6614class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6615 RegisterOperand VecList> 6616: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6617 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6618 "", 6619 []>, Sched<[]> { 6620 bits<3> Pg; 6621 bits<5> Rn; 6622 bits<5> Zt; 6623 bits<4> imm4; 6624 let Inst{31-25} = 0b1010010; 6625 let Inst{24-21} = dtype; 6626 let Inst{20} = nf; 6627 let Inst{19-16} = imm4; 6628 let Inst{15-13} = 0b101; 6629 let Inst{12-10} = Pg; 6630 let Inst{9-5} = Rn; 6631 let Inst{4-0} = Zt; 6632 6633 let mayLoad = 1; 6634 let Uses = !if(nf, [FFR], []); 6635 let Defs = !if(nf, [FFR], []); 6636} 6637 6638multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6639 RegisterOperand listty, ZPRRegOp zprty> { 6640 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6641 6642 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6643 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6644 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6645 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6646 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6647 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6648 6649 // We need a layer of indirection because early machine code passes balk at 6650 // physical register (i.e. FFR) uses that have no previous definition. 6651 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6652 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6653 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6654 } 6655} 6656 6657multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6658 ZPRRegOp zprty> 6659: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6660 6661class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6662: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6663 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6664 "", 6665 []>, Sched<[]> { 6666 bits<5> Zt; 6667 bits<3> Pg; 6668 bits<5> Rn; 6669 bits<4> imm4; 6670 let Inst{31-25} = 0b1010010; 6671 let Inst{24-23} = msz; 6672 let Inst{22-20} = 0b000; 6673 let Inst{19-16} = imm4; 6674 let Inst{15-13} = 0b111; 6675 let Inst{12-10} = Pg; 6676 let Inst{9-5} = Rn; 6677 let Inst{4-0} = Zt; 6678 6679 let mayLoad = 1; 6680} 6681 6682multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6683 ZPRRegOp zprty> { 6684 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6685 6686 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6687 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6688 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6689 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6690 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6691 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6692} 6693 6694class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6695 RegisterOperand gprty> 6696: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6697 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6698 "", 6699 []>, Sched<[]> { 6700 bits<3> Pg; 6701 bits<5> Rm; 6702 bits<5> Rn; 6703 bits<5> Zt; 6704 let Inst{31-25} = 0b1010010; 6705 let Inst{24-23} = msz; 6706 let Inst{22-21} = 0b00; 6707 let Inst{20-16} = Rm; 6708 let Inst{15-13} = 0b110; 6709 let Inst{12-10} = Pg; 6710 let Inst{9-5} = Rn; 6711 let Inst{4-0} = Zt; 6712 6713 let mayLoad = 1; 6714} 6715 6716multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6717 ZPRRegOp zprty, RegisterOperand gprty> { 6718 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6719 6720 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6721 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6722} 6723 6724class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6725: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6726 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6727 bits<5> Zt; 6728 bits<5> Rn; 6729 bits<3> Pg; 6730 bits<4> imm4; 6731 let Inst{31-25} = 0b1010010; 6732 let Inst{24-23} = sz; 6733 let Inst{22-20} = 0; 6734 let Inst{19-16} = imm4; 6735 let Inst{15-13} = 0b001; 6736 let Inst{12-10} = Pg; 6737 let Inst{9-5} = Rn; 6738 let Inst{4-0} = Zt; 6739 6740 let mayLoad = 1; 6741} 6742 6743multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6744 ZPRRegOp zprty> { 6745 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6746 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6747 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6748 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6749 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6750 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6751 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6752} 6753 6754class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6755 RegisterOperand gprty> 6756: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6757 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6758 bits<5> Zt; 6759 bits<3> Pg; 6760 bits<5> Rn; 6761 bits<5> Rm; 6762 let Inst{31-25} = 0b1010010; 6763 let Inst{24-23} = sz; 6764 let Inst{22-21} = 0; 6765 let Inst{20-16} = Rm; 6766 let Inst{15-13} = 0; 6767 let Inst{12-10} = Pg; 6768 let Inst{9-5} = Rn; 6769 let Inst{4-0} = Zt; 6770 6771 let mayLoad = 1; 6772} 6773 6774multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6775 ZPRRegOp zprty, RegisterOperand gprty> { 6776 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6777 6778 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6779 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6780} 6781 6782class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6783 RegisterOperand VecList, Operand immtype> 6784: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6785 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6786 "", 6787 []>, Sched<[]> { 6788 bits<3> Pg; 6789 bits<5> Rn; 6790 bits<5> Zt; 6791 bits<6> imm6; 6792 let Inst{31-25} = 0b1000010; 6793 let Inst{24-23} = dtypeh; 6794 let Inst{22} = 1; 6795 let Inst{21-16} = imm6; 6796 let Inst{15} = 0b1; 6797 let Inst{14-13} = dtypel; 6798 let Inst{12-10} = Pg; 6799 let Inst{9-5} = Rn; 6800 let Inst{4-0} = Zt; 6801 6802 let mayLoad = 1; 6803} 6804 6805multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6806 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6807 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6808 6809 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6810 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6811 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6812 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6813 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6814 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6815} 6816 6817class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6818 RegisterOperand VecList> 6819: I<(outs VecList:$Zt), iops, 6820 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6821 "", 6822 []>, Sched<[]> { 6823 bits<5> Zt; 6824 bits<3> Pg; 6825 bits<5> Rm; 6826 bits<5> Rn; 6827 let Inst{31-25} = 0b1010010; 6828 let Inst{24-21} = dtype; 6829 let Inst{20-16} = Rm; 6830 let Inst{15-14} = 0b01; 6831 let Inst{13} = ff; 6832 let Inst{12-10} = Pg; 6833 let Inst{9-5} = Rn; 6834 let Inst{4-0} = Zt; 6835 6836 let mayLoad = 1; 6837 let Uses = !if(ff, [FFR], []); 6838 let Defs = !if(ff, [FFR], []); 6839} 6840 6841multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6842 ZPRRegOp zprty, RegisterOperand gprty> { 6843 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6844 asm, listty>; 6845 6846 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6847 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6848} 6849 6850multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 6851 ZPRRegOp zprty, RegisterOperand gprty> { 6852 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6853 asm, listty>; 6854 6855 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6856 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6857 6858 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6859 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 6860 6861 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6862 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 6863 6864 // We need a layer of indirection because early machine code passes balk at 6865 // physical register (i.e. FFR) uses that have no previous definition. 6866 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6867 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 6868 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 6869 } 6870} 6871 6872multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 6873 ZPRRegOp zprty> 6874: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 6875 6876class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6877 string asm, Operand immtype> 6878: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 6879 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6880 "", 6881 []>, Sched<[]> { 6882 bits<5> Zt; 6883 bits<3> Pg; 6884 bits<5> Rn; 6885 bits<4> imm4; 6886 let Inst{31-25} = 0b1010010; 6887 let Inst{24-23} = sz; 6888 let Inst{22-21} = nregs; 6889 let Inst{20} = 0; 6890 let Inst{19-16} = imm4; 6891 let Inst{15-13} = 0b111; 6892 let Inst{12-10} = Pg; 6893 let Inst{9-5} = Rn; 6894 let Inst{4-0} = Zt; 6895 6896 let mayLoad = 1; 6897} 6898 6899multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6900 string asm, Operand immtype> { 6901 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 6902 6903 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6904 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6905} 6906 6907class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6908 string asm, RegisterOperand gprty> 6909: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6910 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6911 "", 6912 []>, Sched<[]> { 6913 bits<3> Pg; 6914 bits<5> Rm; 6915 bits<5> Rn; 6916 bits<5> Zt; 6917 let Inst{31-25} = 0b1010010; 6918 let Inst{24-23} = sz; 6919 let Inst{22-21} = nregs; 6920 let Inst{20-16} = Rm; 6921 let Inst{15-13} = 0b110; 6922 let Inst{12-10} = Pg; 6923 let Inst{9-5} = Rn; 6924 let Inst{4-0} = Zt; 6925 6926 let mayLoad = 1; 6927} 6928 6929//===----------------------------------------------------------------------===// 6930// SVE Memory - 32-bit Gather and Unsized Contiguous Group 6931//===----------------------------------------------------------------------===// 6932 6933// bit xs is '1' if offsets are signed 6934// bit scaled is '1' if the offsets are scaled 6935class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 6936 RegisterOperand zprext> 6937: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6938 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 6939 "", 6940 []>, Sched<[]> { 6941 bits<3> Pg; 6942 bits<5> Rn; 6943 bits<5> Zm; 6944 bits<5> Zt; 6945 let Inst{31-25} = 0b1000010; 6946 let Inst{24-23} = opc{3-2}; 6947 let Inst{22} = xs; 6948 let Inst{21} = scaled; 6949 let Inst{20-16} = Zm; 6950 let Inst{15} = 0b0; 6951 let Inst{14-13} = opc{1-0}; 6952 let Inst{12-10} = Pg; 6953 let Inst{9-5} = Rn; 6954 let Inst{4-0} = Zt; 6955 6956 let mayLoad = 1; 6957 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6958 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6959} 6960 6961multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 6962 SDPatternOperator sxtw_op, 6963 SDPatternOperator uxtw_op, 6964 RegisterOperand sxtw_opnd, 6965 RegisterOperand uxtw_opnd, 6966 ValueType vt> { 6967 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 6968 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 6969 6970 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6971 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6972 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6973 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6974 6975 // We need a layer of indirection because early machine code passes balk at 6976 // physical register (i.e. FFR) uses that have no previous definition. 6977 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6978 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6979 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6980 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6981 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6982 } 6983 6984 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6985 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6986 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6987 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6988} 6989 6990multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 6991 SDPatternOperator sxtw_op, 6992 SDPatternOperator uxtw_op, 6993 RegisterOperand sxtw_opnd, 6994 RegisterOperand uxtw_opnd, 6995 ValueType vt> { 6996 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 6997 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 6998 6999 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7000 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7001 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7002 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7003 7004 // We need a layer of indirection because early machine code passes balk at 7005 // physical register (i.e. FFR) uses that have no previous definition. 7006 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7007 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7008 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7009 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7010 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7011 } 7012 7013 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7014 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7015 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7016 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7017} 7018 7019 7020class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7021: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7022 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7023 "", 7024 []>, Sched<[]> { 7025 bits<3> Pg; 7026 bits<5> Zn; 7027 bits<5> Zt; 7028 bits<5> imm5; 7029 let Inst{31-25} = 0b1000010; 7030 let Inst{24-23} = opc{3-2}; 7031 let Inst{22-21} = 0b01; 7032 let Inst{20-16} = imm5; 7033 let Inst{15} = 0b1; 7034 let Inst{14-13} = opc{1-0}; 7035 let Inst{12-10} = Pg; 7036 let Inst{9-5} = Zn; 7037 let Inst{4-0} = Zt; 7038 7039 let mayLoad = 1; 7040 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7041 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7042} 7043 7044multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 7045 SDPatternOperator op, ValueType vt> { 7046 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 7047 7048 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7049 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 7050 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7051 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 7052 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7053 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7054 7055 // We need a layer of indirection because early machine code passes balk at 7056 // physical register (i.e. FFR) uses that have no previous definition. 7057 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7058 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 7059 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 7060 } 7061 7062 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 7063 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7064} 7065 7066class sve_mem_prfm_si<bits<2> msz, string asm> 7067: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 7068 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 7069 "", 7070 []>, Sched<[]> { 7071 bits<5> Rn; 7072 bits<3> Pg; 7073 bits<6> imm6; 7074 bits<4> prfop; 7075 let Inst{31-22} = 0b1000010111; 7076 let Inst{21-16} = imm6; 7077 let Inst{15} = 0b0; 7078 let Inst{14-13} = msz; 7079 let Inst{12-10} = Pg; 7080 let Inst{9-5} = Rn; 7081 let Inst{4} = 0b0; 7082 let Inst{3-0} = prfop; 7083 7084 let hasSideEffects = 1; 7085} 7086 7087multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 7088 def NAME : sve_mem_prfm_si<msz, asm>; 7089 7090 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 7091 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7092} 7093 7094class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 7095: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7096 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 7097 "", 7098 []>, Sched<[]> { 7099 bits<5> Rm; 7100 bits<5> Rn; 7101 bits<3> Pg; 7102 bits<4> prfop; 7103 let Inst{31-25} = 0b1000010; 7104 let Inst{24-23} = opc{2-1}; 7105 let Inst{22-21} = 0b00; 7106 let Inst{20-16} = Rm; 7107 let Inst{15} = 0b1; 7108 let Inst{14} = opc{0}; 7109 let Inst{13} = 0b0; 7110 let Inst{12-10} = Pg; 7111 let Inst{9-5} = Rn; 7112 let Inst{4} = 0b0; 7113 let Inst{3-0} = prfop; 7114 7115 let hasSideEffects = 1; 7116} 7117 7118class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 7119 RegisterOperand zprext> 7120: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7121 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7122 "", 7123 []>, Sched<[]> { 7124 bits<3> Pg; 7125 bits<5> Rn; 7126 bits<5> Zm; 7127 bits<4> prfop; 7128 let Inst{31-23} = 0b100001000; 7129 let Inst{22} = xs; 7130 let Inst{21} = 0b1; 7131 let Inst{20-16} = Zm; 7132 let Inst{15} = 0b0; 7133 let Inst{14-13} = msz; 7134 let Inst{12-10} = Pg; 7135 let Inst{9-5} = Rn; 7136 let Inst{4} = 0b0; 7137 let Inst{3-0} = prfop; 7138 7139 let hasSideEffects = 1; 7140} 7141 7142multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 7143 RegisterOperand sxtw_opnd, 7144 RegisterOperand uxtw_opnd, 7145 SDPatternOperator op_sxtw, 7146 SDPatternOperator op_uxtw> { 7147 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 7148 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 7149 7150 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7151 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7152 7153 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7154 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7155} 7156 7157class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7158: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7159 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7160 "", 7161 []>, Sched<[]> { 7162 bits<3> Pg; 7163 bits<5> Zn; 7164 bits<5> imm5; 7165 bits<4> prfop; 7166 let Inst{31-25} = 0b1000010; 7167 let Inst{24-23} = msz; 7168 let Inst{22-21} = 0b00; 7169 let Inst{20-16} = imm5; 7170 let Inst{15-13} = 0b111; 7171 let Inst{12-10} = Pg; 7172 let Inst{9-5} = Zn; 7173 let Inst{4} = 0b0; 7174 let Inst{3-0} = prfop; 7175} 7176 7177multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7178 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 7179 7180 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7181 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7182 7183 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7184 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7185} 7186 7187class sve_mem_z_fill<string asm> 7188: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 7189 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 7190 "", 7191 []>, Sched<[]> { 7192 bits<5> Rn; 7193 bits<5> Zt; 7194 bits<9> imm9; 7195 let Inst{31-22} = 0b1000010110; 7196 let Inst{21-16} = imm9{8-3}; 7197 let Inst{15-13} = 0b010; 7198 let Inst{12-10} = imm9{2-0}; 7199 let Inst{9-5} = Rn; 7200 let Inst{4-0} = Zt; 7201 7202 let mayLoad = 1; 7203} 7204 7205multiclass sve_mem_z_fill<string asm> { 7206 def NAME : sve_mem_z_fill<asm>; 7207 7208 def : InstAlias<asm # "\t$Zt, [$Rn]", 7209 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 7210} 7211 7212class sve_mem_p_fill<string asm> 7213: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 7214 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 7215 "", 7216 []>, Sched<[]> { 7217 bits<4> Pt; 7218 bits<5> Rn; 7219 bits<9> imm9; 7220 let Inst{31-22} = 0b1000010110; 7221 let Inst{21-16} = imm9{8-3}; 7222 let Inst{15-13} = 0b000; 7223 let Inst{12-10} = imm9{2-0}; 7224 let Inst{9-5} = Rn; 7225 let Inst{4} = 0b0; 7226 let Inst{3-0} = Pt; 7227 7228 let mayLoad = 1; 7229} 7230 7231multiclass sve_mem_p_fill<string asm> { 7232 def NAME : sve_mem_p_fill<asm>; 7233 7234 def : InstAlias<asm # "\t$Pt, [$Rn]", 7235 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 7236} 7237 7238class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 7239 RegisterOperand VecList> 7240: I<(outs VecList:$Zt), iops, 7241 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 7242 "", 7243 []>, Sched<[]> { 7244 bits<3> Pg; 7245 bits<5> Rm; 7246 bits<5> Zn; 7247 bits<5> Zt; 7248 let Inst{31} = 0b1; 7249 let Inst{30} = opc{4}; 7250 let Inst{29-25} = 0b00010; 7251 let Inst{24-23} = opc{3-2}; 7252 let Inst{22-21} = 0b00; 7253 let Inst{20-16} = Rm; 7254 let Inst{15} = 0b1; 7255 let Inst{14-13} = opc{1-0}; 7256 let Inst{12-10} = Pg; 7257 let Inst{9-5} = Zn; 7258 let Inst{4-0} = Zt; 7259 7260 let mayLoad = 1; 7261} 7262 7263multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 7264 SDPatternOperator op, 7265 ValueType vt> { 7266 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 7267 asm, Z_s>; 7268 7269 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7270 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 7271 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7272 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 7273 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7274 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 7275 7276 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 7277 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 7278} 7279 7280multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 7281 SDPatternOperator op, 7282 ValueType vt> { 7283 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 7284 asm, Z_d>; 7285 7286 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7287 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 7288 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7289 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 7290 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7291 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 7292 7293 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 7294 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 7295} 7296 7297//===----------------------------------------------------------------------===// 7298// SVE Memory - 64-bit Gather Group 7299//===----------------------------------------------------------------------===// 7300 7301// bit xs is '1' if offsets are signed 7302// bit scaled is '1' if the offsets are scaled 7303// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7304class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 7305 RegisterOperand zprext> 7306: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7307 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7308 "", 7309 []>, Sched<[]> { 7310 bits<3> Pg; 7311 bits<5> Rn; 7312 bits<5> Zm; 7313 bits<5> Zt; 7314 let Inst{31-25} = 0b1100010; 7315 let Inst{24-23} = opc{3-2}; 7316 let Inst{22} = xs; 7317 let Inst{21} = scaled; 7318 let Inst{20-16} = Zm; 7319 let Inst{15} = lsl; 7320 let Inst{14-13} = opc{1-0}; 7321 let Inst{12-10} = Pg; 7322 let Inst{9-5} = Rn; 7323 let Inst{4-0} = Zt; 7324 7325 let mayLoad = 1; 7326 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7327 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7328} 7329 7330multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 7331 SDPatternOperator sxtw_op, 7332 SDPatternOperator uxtw_op, 7333 RegisterOperand sxtw_opnd, 7334 RegisterOperand uxtw_opnd, 7335 ValueType vt> { 7336 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 7337 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 7338 7339 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7340 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7341 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7342 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7343 7344 // We need a layer of indirection because early machine code passes balk at 7345 // physical register (i.e. FFR) uses that have no previous definition. 7346 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7347 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7348 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7349 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7350 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7351 } 7352 7353 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7354 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7355 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7356 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7357} 7358 7359multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 7360 SDPatternOperator sxtw_op, 7361 SDPatternOperator uxtw_op, 7362 RegisterOperand sxtw_opnd, 7363 RegisterOperand uxtw_opnd, 7364 ValueType vt> { 7365 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 7366 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 7367 7368 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7369 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7370 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7371 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7372 7373 // We need a layer of indirection because early machine code passes balk at 7374 // physical register (i.e. FFR) uses that have no previous definition. 7375 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7376 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7377 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7378 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7379 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7380 } 7381 7382 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7383 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7384 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7385 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7386} 7387 7388multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 7389 SDPatternOperator op, 7390 RegisterOperand zprext, ValueType vt> { 7391 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 7392 7393 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7394 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 7395 7396 // We need a layer of indirection because early machine code passes balk at 7397 // physical register (i.e. FFR) uses that have no previous definition. 7398 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7399 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 7400 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7401 } 7402 7403 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7404 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7405} 7406 7407multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 7408 SDPatternOperator op, ValueType vt> { 7409 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 7410 7411 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7412 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 7413 7414 // We need a layer of indirection because early machine code passes balk at 7415 // physical register (i.e. FFR) uses that have no previous definition. 7416 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7417 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 7418 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 7419 } 7420 7421 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7422 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7423} 7424 7425class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7426: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7427 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7428 "", 7429 []>, Sched<[]> { 7430 bits<3> Pg; 7431 bits<5> Zn; 7432 bits<5> Zt; 7433 bits<5> imm5; 7434 let Inst{31-25} = 0b1100010; 7435 let Inst{24-23} = opc{3-2}; 7436 let Inst{22-21} = 0b01; 7437 let Inst{20-16} = imm5; 7438 let Inst{15} = 0b1; 7439 let Inst{14-13} = opc{1-0}; 7440 let Inst{12-10} = Pg; 7441 let Inst{9-5} = Zn; 7442 let Inst{4-0} = Zt; 7443 7444 let mayLoad = 1; 7445 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7446 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7447} 7448 7449multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7450 SDPatternOperator op, ValueType vt> { 7451 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7452 7453 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7454 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7455 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7456 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7457 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7458 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7459 7460 // We need a layer of indirection because early machine code passes balk at 7461 // physical register (i.e. FFR) uses that have no previous definition. 7462 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7463 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7464 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7465 } 7466 7467 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7468 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7469} 7470 7471// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7472class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7473 RegisterOperand zprext> 7474: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7475 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7476 "", 7477 []>, Sched<[]> { 7478 bits<3> Pg; 7479 bits<5> Rn; 7480 bits<5> Zm; 7481 bits<4> prfop; 7482 let Inst{31-23} = 0b110001000; 7483 let Inst{22} = xs; 7484 let Inst{21} = 0b1; 7485 let Inst{20-16} = Zm; 7486 let Inst{15} = lsl; 7487 let Inst{14-13} = msz; 7488 let Inst{12-10} = Pg; 7489 let Inst{9-5} = Rn; 7490 let Inst{4} = 0b0; 7491 let Inst{3-0} = prfop; 7492 7493 let hasSideEffects = 1; 7494} 7495 7496multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7497 RegisterOperand sxtw_opnd, 7498 RegisterOperand uxtw_opnd, 7499 SDPatternOperator op_sxtw, 7500 SDPatternOperator op_uxtw> { 7501 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7502 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7503 7504 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7505 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7506 7507 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7508 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7509 7510} 7511 7512multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7513 RegisterOperand zprext, SDPatternOperator frag> { 7514 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7515 7516 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7517 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7518 7519} 7520 7521class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7522: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7523 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7524 "", 7525 []>, Sched<[]> { 7526 bits<3> Pg; 7527 bits<5> Zn; 7528 bits<5> imm5; 7529 bits<4> prfop; 7530 let Inst{31-25} = 0b1100010; 7531 let Inst{24-23} = msz; 7532 let Inst{22-21} = 0b00; 7533 let Inst{20-16} = imm5; 7534 let Inst{15-13} = 0b111; 7535 let Inst{12-10} = Pg; 7536 let Inst{9-5} = Zn; 7537 let Inst{4} = 0b0; 7538 let Inst{3-0} = prfop; 7539 7540 let hasSideEffects = 1; 7541} 7542 7543multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7544 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7545 7546 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7547 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7548 7549 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7550 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7551} 7552 7553//===----------------------------------------------------------------------===// 7554// SVE Compute Vector Address Group 7555//===----------------------------------------------------------------------===// 7556 7557class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7558 ZPRRegOp zprty, RegisterOperand zprext> 7559: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7560 asm, "\t$Zd, [$Zn, $Zm]", 7561 "", 7562 []>, Sched<[]> { 7563 bits<5> Zd; 7564 bits<5> Zn; 7565 bits<5> Zm; 7566 let Inst{31-24} = 0b00000100; 7567 let Inst{23-22} = opc; 7568 let Inst{21} = 0b1; 7569 let Inst{20-16} = Zm; 7570 let Inst{15-12} = 0b1010; 7571 let Inst{11-10} = msz; 7572 let Inst{9-5} = Zn; 7573 let Inst{4-0} = Zd; 7574} 7575 7576multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7577 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7578 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7579 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7580 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7581} 7582 7583multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7584 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7585 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7586 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7587 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7588} 7589 7590multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7591 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7592 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7593 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7594 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7595} 7596 7597multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7598 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7599 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7600 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7601 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7602} 7603 7604//===----------------------------------------------------------------------===// 7605// SVE Integer Misc - Unpredicated Group 7606//===----------------------------------------------------------------------===// 7607 7608class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7609: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7610 asm, "\t$Zd, $Zn, $Zm", 7611 "", 7612 []>, Sched<[]> { 7613 bits<5> Zd; 7614 bits<5> Zm; 7615 bits<5> Zn; 7616 let Inst{31-24} = 0b00000100; 7617 let Inst{23-22} = sz; 7618 let Inst{21} = 0b1; 7619 let Inst{20-16} = Zm; 7620 let Inst{15-10} = 0b101100; 7621 let Inst{9-5} = Zn; 7622 let Inst{4-0} = Zd; 7623} 7624 7625multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7626 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7627 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7628 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7629 7630 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7631 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7632 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7633} 7634 7635class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7636: I<(outs zprty:$Zd), (ins zprty:$Zn), 7637 asm, "\t$Zd, $Zn", 7638 "", 7639 []>, Sched<[]> { 7640 bits<5> Zd; 7641 bits<5> Zn; 7642 let Inst{31-24} = 0b00000100; 7643 let Inst{23-22} = opc{7-6}; 7644 let Inst{21} = 0b1; 7645 let Inst{20-16} = opc{5-1}; 7646 let Inst{15-11} = 0b10111; 7647 let Inst{10} = opc{0}; 7648 let Inst{9-5} = Zn; 7649 let Inst{4-0} = Zd; 7650} 7651 7652multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7653 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7654 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7655 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7656 7657 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7658 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7659 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7660} 7661 7662//===----------------------------------------------------------------------===// 7663// SVE Integer Reduction Group 7664//===----------------------------------------------------------------------===// 7665 7666class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7667 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7668: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7669 asm, "\t$Vd, $Pg, $Zn", 7670 "", 7671 []>, Sched<[]> { 7672 bits<3> Pg; 7673 bits<5> Vd; 7674 bits<5> Zn; 7675 let Inst{31-24} = 0b00000100; 7676 let Inst{23-22} = sz8_32; 7677 let Inst{21} = 0b0; 7678 let Inst{20-19} = fmt; 7679 let Inst{18-16} = opc; 7680 let Inst{15-13} = 0b001; 7681 let Inst{12-10} = Pg; 7682 let Inst{9-5} = Zn; 7683 let Inst{4-0} = Vd; 7684} 7685 7686multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7687 SDPatternOperator op> { 7688 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7689 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7690 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7691 7692 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7693 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7694 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7695} 7696 7697multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7698 SDPatternOperator op> { 7699 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7700 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7701 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7702 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7703 7704 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7705 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7706 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7707 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7708} 7709 7710multiclass sve_int_reduce_1<bits<3> opc, string asm, 7711 SDPatternOperator op> { 7712 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7713 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7714 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7715 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7716 7717 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7718 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7719 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7720 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7721} 7722 7723multiclass sve_int_reduce_2<bits<3> opc, string asm, 7724 SDPatternOperator op> { 7725 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 7726 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 7727 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 7728 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 7729 7730 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7731 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7732 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7733 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7734} 7735 7736class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7737 ZPRRegOp zprty, string pg_suffix, dag iops> 7738: I<(outs zprty:$Zd), iops, 7739 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7740 "", 7741 []>, Sched<[]> { 7742 bits<3> Pg; 7743 bits<5> Zd; 7744 bits<5> Zn; 7745 let Inst{31-24} = 0b00000100; 7746 let Inst{23-22} = sz8_32; 7747 let Inst{21-19} = 0b010; 7748 let Inst{18-16} = opc; 7749 let Inst{15-13} = 0b001; 7750 let Inst{12-10} = Pg; 7751 let Inst{9-5} = Zn; 7752 let Inst{4-0} = Zd; 7753 7754 let ElementSize = zprty.ElementSize; 7755} 7756 7757multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7758let Constraints = "$Zd = $_Zd" in { 7759 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7760 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7761 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7762 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7763 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7764 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7765 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7766 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7767} 7768} 7769 7770multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7771 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7772 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7773 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7774 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7775 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7776 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7777 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7778 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7779} 7780 7781//===----------------------------------------------------------------------===// 7782// SVE Propagate Break Group 7783//===----------------------------------------------------------------------===// 7784 7785class sve_int_brkp<bits<2> opc, string asm> 7786: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7787 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7788 "", 7789 []>, Sched<[]> { 7790 bits<4> Pd; 7791 bits<4> Pg; 7792 bits<4> Pm; 7793 bits<4> Pn; 7794 let Inst{31-24} = 0b00100101; 7795 let Inst{23} = 0b0; 7796 let Inst{22} = opc{1}; 7797 let Inst{21-20} = 0b00; 7798 let Inst{19-16} = Pm; 7799 let Inst{15-14} = 0b11; 7800 let Inst{13-10} = Pg; 7801 let Inst{9} = 0b0; 7802 let Inst{8-5} = Pn; 7803 let Inst{4} = opc{0}; 7804 let Inst{3-0} = Pd; 7805 7806 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7807} 7808 7809multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7810 def NAME : sve_int_brkp<opc, asm>; 7811 7812 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7813} 7814 7815 7816//===----------------------------------------------------------------------===// 7817// SVE Partition Break Group 7818//===----------------------------------------------------------------------===// 7819 7820class sve_int_brkn<bit S, string asm> 7821: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7822 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7823 "", 7824 []>, Sched<[]> { 7825 bits<4> Pdm; 7826 bits<4> Pg; 7827 bits<4> Pn; 7828 let Inst{31-23} = 0b001001010; 7829 let Inst{22} = S; 7830 let Inst{21-14} = 0b01100001; 7831 let Inst{13-10} = Pg; 7832 let Inst{9} = 0b0; 7833 let Inst{8-5} = Pn; 7834 let Inst{4} = 0b0; 7835 let Inst{3-0} = Pdm; 7836 7837 let Constraints = "$Pdm = $_Pdm"; 7838 let Defs = !if(S, [NZCV], []); 7839} 7840 7841multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7842 def NAME : sve_int_brkn<opc, asm>; 7843 7844 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7845} 7846 7847class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 7848: I<(outs PPR8:$Pd), iops, 7849 asm, "\t$Pd, $Pg"#suffix#", $Pn", 7850 "", 7851 []>, Sched<[]> { 7852 bits<4> Pd; 7853 bits<4> Pg; 7854 bits<4> Pn; 7855 let Inst{31-24} = 0b00100101; 7856 let Inst{23-22} = opc{2-1}; 7857 let Inst{21-14} = 0b01000001; 7858 let Inst{13-10} = Pg; 7859 let Inst{9} = 0b0; 7860 let Inst{8-5} = Pn; 7861 let Inst{4} = opc{0}; 7862 let Inst{3-0} = Pd; 7863 7864 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 7865 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7866 7867} 7868 7869multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 7870 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 7871 7872 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7873} 7874 7875multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 7876 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 7877 7878 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7879} 7880 7881//===----------------------------------------------------------------------===// 7882// SVE2 String Processing Group 7883//===----------------------------------------------------------------------===// 7884 7885class sve2_char_match<bit sz, bit opc, string asm, 7886 PPRRegOp pprty, ZPRRegOp zprty> 7887: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7888 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 7889 "", 7890 []>, Sched<[]> { 7891 bits<4> Pd; 7892 bits<3> Pg; 7893 bits<5> Zm; 7894 bits<5> Zn; 7895 let Inst{31-23} = 0b010001010; 7896 let Inst{22} = sz; 7897 let Inst{21} = 0b1; 7898 let Inst{20-16} = Zm; 7899 let Inst{15-13} = 0b100; 7900 let Inst{12-10} = Pg; 7901 let Inst{9-5} = Zn; 7902 let Inst{4} = opc; 7903 let Inst{3-0} = Pd; 7904 7905 let Defs = [NZCV]; 7906 let isPTestLike = 1; 7907} 7908 7909multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 7910 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 7911 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 7912 7913 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 7914 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7915} 7916 7917//===----------------------------------------------------------------------===// 7918// SVE2 Histogram Computation - Segment Group 7919//===----------------------------------------------------------------------===// 7920 7921class sve2_hist_gen_segment<string asm, SDPatternOperator op> 7922: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 7923 asm, "\t$Zd, $Zn, $Zm", 7924 "", 7925 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 7926 bits<5> Zd; 7927 bits<5> Zn; 7928 bits<5> Zm; 7929 let Inst{31-21} = 0b01000101001; 7930 let Inst{20-16} = Zm; 7931 let Inst{15-10} = 0b101000; 7932 let Inst{9-5} = Zn; 7933 let Inst{4-0} = Zd; 7934} 7935 7936//===----------------------------------------------------------------------===// 7937// SVE2 Histogram Computation - Vector Group 7938//===----------------------------------------------------------------------===// 7939 7940class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 7941: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7942 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 7943 "", 7944 []>, Sched<[]> { 7945 bits<5> Zd; 7946 bits<5> Zn; 7947 bits<3> Pg; 7948 bits<5> Zm; 7949 let Inst{31-23} = 0b010001011; 7950 let Inst{22} = sz; 7951 let Inst{21} = 0b1; 7952 let Inst{20-16} = Zm; 7953 let Inst{15-13} = 0b110; 7954 let Inst{12-10} = Pg; 7955 let Inst{9-5} = Zn; 7956 let Inst{4-0} = Zd; 7957} 7958 7959multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 7960 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 7961 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 7962 7963 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7964 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7965} 7966 7967//===----------------------------------------------------------------------===// 7968// SVE2 Crypto Extensions Group 7969//===----------------------------------------------------------------------===// 7970 7971class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 7972: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7973 asm, "\t$Zd, $Zn, $Zm", 7974 "", 7975 []>, Sched<[]> { 7976 bits<5> Zd; 7977 bits<5> Zn; 7978 bits<5> Zm; 7979 let Inst{31-21} = 0b01000101001; 7980 let Inst{20-16} = Zm; 7981 let Inst{15-11} = 0b11110; 7982 let Inst{10} = opc; 7983 let Inst{9-5} = Zn; 7984 let Inst{4-0} = Zd; 7985} 7986 7987multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 7988 SDPatternOperator op, ValueType vt> { 7989 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 7990 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7991} 7992 7993class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 7994: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 7995 asm, "\t$Zdn, $_Zdn, $Zm", 7996 "", 7997 []>, Sched<[]> { 7998 bits<5> Zdn; 7999 bits<5> Zm; 8000 let Inst{31-17} = 0b010001010010001; 8001 let Inst{16} = opc{1}; 8002 let Inst{15-11} = 0b11100; 8003 let Inst{10} = opc{0}; 8004 let Inst{9-5} = Zm; 8005 let Inst{4-0} = Zdn; 8006 8007 let Constraints = "$Zdn = $_Zdn"; 8008} 8009 8010multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 8011 SDPatternOperator op, ValueType vt> { 8012 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 8013 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8014} 8015 8016class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 8017: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 8018 asm, "\t$Zdn, $_Zdn", 8019 "", 8020 []>, Sched<[]> { 8021 bits<5> Zdn; 8022 let Inst{31-11} = 0b010001010010000011100; 8023 let Inst{10} = opc; 8024 let Inst{9-5} = 0b00000; 8025 let Inst{4-0} = Zdn; 8026 8027 let Constraints = "$Zdn = $_Zdn"; 8028} 8029 8030multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 8031 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 8032 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 8033} 8034 8035//===----------------------------------------------------------------------===// 8036// SVE BFloat16 Group 8037//===----------------------------------------------------------------------===// 8038 8039class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 8040: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 8041 bits<5> Zda; 8042 bits<5> Zn; 8043 let Inst{31-21} = 0b01100100011; 8044 let Inst{15-14} = opc; 8045 let Inst{13-10} = 0b0000; 8046 let Inst{9-5} = Zn; 8047 let Inst{4-0} = Zda; 8048 8049 let Constraints = "$Zda = $_Zda"; 8050 let DestructiveInstType = DestructiveOther; 8051 let ElementSize = ElementSizeH; 8052} 8053 8054class sve_bfloat_dot<string asm> 8055: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 8056 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 8057 bits<5> Zm; 8058 let Inst{20-16} = Zm; 8059} 8060 8061multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 8062 def NAME : sve_bfloat_dot<asm>; 8063 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8064} 8065 8066class sve_bfloat_dot_indexed<string asm> 8067: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8068 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 8069 bits<2> iop; 8070 bits<3> Zm; 8071 let Inst{20-19} = iop; 8072 let Inst{18-16} = Zm; 8073} 8074 8075multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 8076 def NAME : sve_bfloat_dot_indexed<asm>; 8077 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 8078} 8079 8080class sve_bfloat_matmul<string asm> 8081: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8082 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8083 bits<5> Zm; 8084 bits<5> Zda; 8085 bits<5> Zn; 8086 let Inst{31-21} = 0b01100100011; 8087 let Inst{20-16} = Zm; 8088 let Inst{15-10} = 0b111001; 8089 let Inst{9-5} = Zn; 8090 let Inst{4-0} = Zda; 8091 8092 let Constraints = "$Zda = $_Zda"; 8093 let DestructiveInstType = DestructiveOther; 8094 let ElementSize = ElementSizeH; 8095} 8096 8097multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 8098 def NAME : sve_bfloat_matmul<asm>; 8099 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8100} 8101 8102class sve_bfloat_matmul_longvecl<bit BT, string asm> 8103: sve_bfloat_matmul<asm> { 8104 let Inst{23} = 0b1; 8105 let Inst{14-13} = 0b00; 8106 let Inst{10} = BT; 8107} 8108 8109multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 8110 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 8111 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8112} 8113 8114class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 8115: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8116 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 8117 bits<3> iop; 8118 bits<3> Zm; 8119 let Inst{23} = 0b1; 8120 let Inst{20-19} = iop{2-1}; 8121 let Inst{18-16} = Zm; 8122 let Inst{11} = iop{0}; 8123 let Inst{10} = BT; 8124} 8125 8126multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 8127 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 8128 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 8129} 8130 8131class sve_bfloat_convert<bit N, string asm> 8132: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 8133 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 8134 bits<5> Zd; 8135 bits<3> Pg; 8136 bits<5> Zn; 8137 let Inst{31-25} = 0b0110010; 8138 let Inst{24} = N; 8139 let Inst{23-13} = 0b10001010101; 8140 let Inst{12-10} = Pg; 8141 let Inst{9-5} = Zn; 8142 let Inst{4-0} = Zd; 8143 8144 let Constraints = "$Zd = $_Zd"; 8145 let DestructiveInstType = DestructiveOther; 8146 let hasSideEffects = 1; 8147 let ElementSize = ElementSizeS; 8148} 8149 8150multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 8151 def NAME : sve_bfloat_convert<N, asm>; 8152 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 8153} 8154 8155//===----------------------------------------------------------------------===// 8156// SVE Integer Matrix Multiply Group 8157//===----------------------------------------------------------------------===// 8158 8159class sve_int_matmul<bits<2> uns, string asm> 8160: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8161 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8162 bits<5> Zda; 8163 bits<5> Zn; 8164 bits<5> Zm; 8165 let Inst{31-24} = 0b01000101; 8166 let Inst{23-22} = uns; 8167 let Inst{21} = 0; 8168 let Inst{20-16} = Zm; 8169 let Inst{15-10} = 0b100110; 8170 let Inst{9-5} = Zn; 8171 let Inst{4-0} = Zda; 8172 8173 let Constraints = "$Zda = $_Zda"; 8174 let DestructiveInstType = DestructiveOther; 8175 let ElementSize = ZPR32.ElementSize; 8176} 8177 8178multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 8179 def NAME : sve_int_matmul<uns, asm>; 8180 8181 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8182} 8183 8184//===----------------------------------------------------------------------===// 8185// SVE Integer Dot Product Mixed Sign Group 8186//===----------------------------------------------------------------------===// 8187 8188class sve_int_dot_mixed<string asm> 8189: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8190 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8191 bits<5> Zda; 8192 bits<5> Zn; 8193 bits<5> Zm; 8194 let Inst{31-21} = 0b01000100100; 8195 let Inst{20-16} = Zm; 8196 let Inst{15-10} = 0b011110; 8197 let Inst{9-5} = Zn; 8198 let Inst{4-0} = Zda; 8199 8200 let Constraints = "$Zda = $_Zda"; 8201 let DestructiveInstType = DestructiveOther; 8202 let ElementSize = ZPR32.ElementSize; 8203} 8204 8205multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 8206 def NAME : sve_int_dot_mixed<asm>; 8207 8208 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8209} 8210 8211//===----------------------------------------------------------------------===// 8212// SVE Integer Dot Product Mixed Sign - Indexed Group 8213//===----------------------------------------------------------------------===// 8214 8215class sve_int_dot_mixed_indexed<bit U, string asm> 8216: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 8217 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 8218 bits<5> Zda; 8219 bits<5> Zn; 8220 bits<3> Zm; 8221 bits<2> idx; 8222 let Inst{31-21} = 0b01000100101; 8223 let Inst{20-19} = idx; 8224 let Inst{18-16} = Zm; 8225 let Inst{15-11} = 0b00011; 8226 let Inst{10} = U; 8227 let Inst{9-5} = Zn; 8228 let Inst{4-0} = Zda; 8229 8230 let Constraints = "$Zda = $_Zda"; 8231 let DestructiveInstType = DestructiveOther; 8232 let ElementSize = ZPR32.ElementSize; 8233} 8234 8235multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 8236 def NAME : sve_int_dot_mixed_indexed<U, asm>; 8237 8238 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8239} 8240 8241//===----------------------------------------------------------------------===// 8242// SVE Floating Point Matrix Multiply Accumulate Group 8243//===----------------------------------------------------------------------===// 8244 8245class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 8246: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 8247 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8248 bits<5> Zda; 8249 bits<5> Zn; 8250 bits<5> Zm; 8251 let Inst{31-23} = 0b011001001; 8252 let Inst{22} = sz; 8253 let Inst{21} = 1; 8254 let Inst{20-16} = Zm; 8255 let Inst{15-10} = 0b111001; 8256 let Inst{9-5} = Zn; 8257 let Inst{4-0} = Zda; 8258 8259 let Constraints = "$Zda = $_Zda"; 8260 let DestructiveInstType = DestructiveOther; 8261 let ElementSize = zprty.ElementSize; 8262} 8263 8264multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 8265 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 8266 8267 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 8268} 8269 8270//===----------------------------------------------------------------------===// 8271// SVE Memory - Contiguous Load And Replicate 256-bit Group 8272//===----------------------------------------------------------------------===// 8273 8274class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 8275: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 8276 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 8277 bits<5> Zt; 8278 bits<5> Rn; 8279 bits<3> Pg; 8280 bits<4> imm4; 8281 let Inst{31-25} = 0b1010010; 8282 let Inst{24-23} = sz; 8283 let Inst{22-20} = 0b010; 8284 let Inst{19-16} = imm4; 8285 let Inst{15-13} = 0b001; 8286 let Inst{12-10} = Pg; 8287 let Inst{9-5} = Rn; 8288 let Inst{4-0} = Zt; 8289 8290 let mayLoad = 1; 8291} 8292 8293multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 8294 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 8295 def NAME : sve_mem_ldor_si<sz, asm, listty>; 8296 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8297 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 8298 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8299 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 8300 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 8301 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 8302 8303 // Base addressing mode 8304 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 8305 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 8306 let AddedComplexity = 2 in { 8307 // Reg + Imm addressing mode 8308 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 8309 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 8310 } 8311} 8312 8313class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 8314 RegisterOperand gprty> 8315: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 8316 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 8317 bits<5> Zt; 8318 bits<3> Pg; 8319 bits<5> Rn; 8320 bits<5> Rm; 8321 let Inst{31-25} = 0b1010010; 8322 let Inst{24-23} = sz; 8323 let Inst{22-21} = 0b01; 8324 let Inst{20-16} = Rm; 8325 let Inst{15-13} = 0; 8326 let Inst{12-10} = Pg; 8327 let Inst{9-5} = Rn; 8328 let Inst{4-0} = Zt; 8329 8330 let mayLoad = 1; 8331} 8332 8333multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 8334 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 8335 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 8336 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 8337 8338 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 8339 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 8340 8341 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 8342 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 8343} 8344 8345//===----------------------------------------------------------------------===// 8346// SVE Interleave 128-bit Elements Group 8347//===----------------------------------------------------------------------===// 8348 8349class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 8350: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 8351 asm, "\t$Zd, $Zn, $Zm", 8352 "", 8353 []>, Sched<[]> { 8354 bits<5> Zd; 8355 bits<5> Zm; 8356 bits<5> Zn; 8357 let Inst{31-21} = 0b00000101101; 8358 let Inst{20-16} = Zm; 8359 let Inst{15-13} = 0b000; 8360 let Inst{12-11} = opc; 8361 let Inst{10} = P; 8362 let Inst{9-5} = Zn; 8363 let Inst{4-0} = Zd; 8364} 8365 8366multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 8367 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 8368 8369 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8370 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 8371 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 8372 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 8373 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 8374 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 8375 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 8376 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 8377} 8378 8379/// Addressing modes 8380def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 8381def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 8382 8383def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>; 8384def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>; 8385def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>; 8386def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>; 8387 8388// Predicated pseudo floating point two operand instructions. 8389multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 8390 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8391 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8392 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8393 8394 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8395 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8396 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8397 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8398 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8399 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8400} 8401 8402// Predicated pseudo integer two operand instructions. 8403multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 8404 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8405 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8406 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8407 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8408 8409 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8410 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8411 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8412 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8413} 8414 8415// As sve_int_bin_pred but when only i32 and i64 vector types are required. 8416multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 8417 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8418 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8419 8420 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8421 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8422} 8423 8424// Predicated pseudo integer two operand instructions. Second operand is an 8425// immediate specified by imm_[bhsd]. 8426multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 8427 ComplexPattern imm_b, ComplexPattern imm_h, 8428 ComplexPattern imm_s, ComplexPattern imm_d> { 8429 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 8430 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 8431 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 8432 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 8433 8434 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 8435 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 8436 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 8437 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8438} 8439 8440multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> { 8441 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8442 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8443 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8444 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8445 8446 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8447 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8448 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8449 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8450} 8451 8452