1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVELogicalImm8Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>; 203def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>; 204def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>; 205def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 206 207def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>; 208 209def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 210def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 211def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 212def SVEArithUImm64Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>; 213def SVEArithSImmPat : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 214 215def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 216def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 217def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 218def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>; 219def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 220def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 221def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 222def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>; 223 224class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 225 let Name = "SVEExactFPImmOperand" # Suffix; 226 let DiagnosticType = "Invalid" # Name; 227 let ParserMethod = "tryParseFPImm<false>"; 228 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 229 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 230} 231 232class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 233 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 234 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 235} 236 237def sve_fpimm_half_one 238 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 239 "AArch64ExactFPImm::one">; 240def sve_fpimm_half_two 241 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 242 "AArch64ExactFPImm::two">; 243def sve_fpimm_zero_one 244 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 245 "AArch64ExactFPImm::one">; 246 247def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 248 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 249}]> { 250 let ParserMatchClass = Imm1_16Operand; 251 let EncoderMethod = "getSVEIncDecImm"; 252 let DecoderMethod = "DecodeSVEIncDecImm"; 253} 254 255// This allows i32 immediate extraction from i64 based arithmetic. 256def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 257def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">; 258 259//===----------------------------------------------------------------------===// 260// SVE PTrue - These are used extensively throughout the pattern matching so 261// it's important we define them first. 262//===----------------------------------------------------------------------===// 263 264class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 265 ValueType vt, SDPatternOperator op> 266: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 267 asm, "\t$Pd, $pattern", 268 "", 269 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 270 bits<4> Pd; 271 bits<5> pattern; 272 let Inst{31-24} = 0b00100101; 273 let Inst{23-22} = sz8_64; 274 let Inst{21-19} = 0b011; 275 let Inst{18-17} = opc{2-1}; 276 let Inst{16} = opc{0}; 277 let Inst{15-10} = 0b111000; 278 let Inst{9-5} = pattern; 279 let Inst{4} = 0b0; 280 let Inst{3-0} = Pd; 281 282 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 283 let ElementSize = pprty.ElementSize; 284 let isReMaterializable = 1; 285} 286 287multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 288 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 289 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 290 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 291 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 292 293 def : InstAlias<asm # "\t$Pd", 294 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 295 def : InstAlias<asm # "\t$Pd", 296 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 297 def : InstAlias<asm # "\t$Pd", 298 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 299 def : InstAlias<asm # "\t$Pd", 300 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 301} 302 303def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 304def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 305 306let Predicates = [HasSVE] in { 307 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 308 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 309} 310 311//===----------------------------------------------------------------------===// 312// SVE pattern match helpers. 313//===----------------------------------------------------------------------===// 314 315class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 316 Instruction inst> 317: Pat<(vtd (op vt1:$Op1)), 318 (inst $Op1)>; 319 320class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 321 ValueType vts, Instruction inst> 322: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 323 (inst $Op3, $Op1, $Op2)>; 324 325// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 326// type of rounding. This is matched by timm0_1 in pattern below and ignored. 327class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 328 ValueType vts, Instruction inst> 329: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 330 (inst $Op3, $Op1, $Op2)>; 331 332class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 333 ValueType it, ComplexPattern cpx, Instruction inst> 334 : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))), 335 (inst $Op1, i32:$imm, i32:$shift)>; 336 337class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 338 ValueType it, ComplexPattern cpx, Instruction inst> 339 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))), 340 (inst $Op1, i32:$imm, i32:$shift)>; 341 342class SVE_1_Op_Imm_Arith_Pred_Pat<ValueType vt, ValueType pt, SDPatternOperator op, 343 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 344 : Pat<(vt (op (pt (AArch64ptrue 31)), (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))), 345 (inst $Op1, i32:$imm)>; 346 347class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 348 ValueType it, ComplexPattern cpx, Instruction inst> 349 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))), 350 (inst $Op1, i64:$imm)>; 351 352class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 353 ValueType vt2, Instruction inst> 354: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 355 (inst $Op1, $Op2)>; 356 357class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 358 ValueType pt, ValueType vt1, ValueType vt2, 359 Instruction inst> 360: Pat<(vtd (op (pt (AArch64ptrue 31)), vt1:$Op1, vt2:$Op2)), 361 (inst $Op1, $Op2)>; 362 363class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 364 ValueType vt2, ValueType vt3, Instruction inst> 365: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 366 (inst $Op1, $Op2, $Op3)>; 367 368class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 369 ValueType vt2, ValueType vt3, ValueType vt4, 370 Instruction inst> 371: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 372 (inst $Op1, $Op2, $Op3, $Op4)>; 373 374class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 375 ValueType vt2, Operand ImmTy, Instruction inst> 376: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 377 (inst $Op1, ImmTy:$Op2)>; 378 379class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 380 ValueType vt2, ValueType vt3, Operand ImmTy, 381 Instruction inst> 382: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 383 (inst $Op1, $Op2, ImmTy:$Op3)>; 384 385class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 386 ValueType vt2, ValueType vt3, ValueType vt4, 387 Operand ImmTy, Instruction inst> 388: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 389 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 390 391def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>; 392def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>; 393 394let AddedComplexity = 1 in { 395class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 396 ValueType vt2, ValueType vt3, Instruction inst> 397: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 398 (inst $Op1, $Op2, $Op3)>; 399 400class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 401 ValueType vt1, ValueType vt2, 402 Operand vt3, Instruction inst> 403: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 404 (inst $Op1, $Op2, vt3:$Op3)>; 405} 406 407// 408// Common but less generic patterns. 409// 410 411class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 412 Instruction inst, Instruction ptrue> 413: Pat<(vtd (op vt1:$Op1)), 414 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 415 416class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 417 ValueType vt2, Instruction inst, Instruction ptrue> 418: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 419 (inst (ptrue 31), $Op1, $Op2)>; 420 421class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 422 ValueType inreg_vt, Instruction inst> 423: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 424 (inst $PassThru, $Pg, $Src)>; 425 426class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 427 ValueType pt, ValueType it, 428 ComplexPattern cast, Instruction inst> 429: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))), 430 (inst $Pg, $Rn, i32:$imm)>; 431 432class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 433 ValueType pt, ValueType it, 434 ComplexPattern cast, Instruction inst> 435: Pat<(vt (op (pt (AArch64ptrue 31)), vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))), 436 (inst $Rn, i32:$imm)>; 437 438// 439// Pseudo -> Instruction mappings 440// 441def getSVEPseudoMap : InstrMapping { 442 let FilterClass = "SVEPseudo2Instr"; 443 let RowFields = ["PseudoName"]; 444 let ColFields = ["IsInstr"]; 445 let KeyCol = ["0"]; 446 let ValueCols = [["1"]]; 447} 448 449class SVEPseudo2Instr<string name, bit instr> { 450 string PseudoName = name; 451 bit IsInstr = instr; 452} 453 454// Lookup e.g. DIV -> DIVR 455def getSVERevInstr : InstrMapping { 456 let FilterClass = "SVEInstr2Rev"; 457 let RowFields = ["InstrName"]; 458 let ColFields = ["isReverseInstr"]; 459 let KeyCol = ["0"]; 460 let ValueCols = [["1"]]; 461} 462 463// Lookup e.g. DIVR -> DIV 464def getSVENonRevInstr : InstrMapping { 465 let FilterClass = "SVEInstr2Rev"; 466 let RowFields = ["InstrName"]; 467 let ColFields = ["isReverseInstr"]; 468 let KeyCol = ["1"]; 469 let ValueCols = [["0"]]; 470} 471 472class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 473 string InstrName = !if(name1IsReverseInstr, name1, name2); 474 bit isReverseInstr = name1IsReverseInstr; 475} 476 477// 478// Pseudos for destructive operands 479// 480let hasNoSchedulingInfo = 1 in { 481 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 482 FalseLanesEnum flags = FalseLanesNone> 483 : SVEPseudo2Instr<name, 0>, 484 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 485 let FalseLanes = flags; 486 } 487 488 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 489 FalseLanesEnum flags = FalseLanesNone> 490 : SVEPseudo2Instr<name, 0>, 491 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 492 let FalseLanes = flags; 493 } 494} 495 496//===----------------------------------------------------------------------===// 497// SVE Predicate Misc Group 498//===----------------------------------------------------------------------===// 499 500class sve_int_pfalse<bits<6> opc, string asm> 501: I<(outs PPR8:$Pd), (ins), 502 asm, "\t$Pd", 503 "", 504 []>, Sched<[]> { 505 bits<4> Pd; 506 let Inst{31-24} = 0b00100101; 507 let Inst{23-22} = opc{5-4}; 508 let Inst{21-19} = 0b011; 509 let Inst{18-16} = opc{3-1}; 510 let Inst{15-10} = 0b111001; 511 let Inst{9} = opc{0}; 512 let Inst{8-4} = 0b00000; 513 let Inst{3-0} = Pd; 514 515 let isReMaterializable = 1; 516} 517 518class sve_int_ptest<bits<6> opc, string asm> 519: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 520 asm, "\t$Pg, $Pn", 521 "", 522 []>, Sched<[]> { 523 bits<4> Pg; 524 bits<4> Pn; 525 let Inst{31-24} = 0b00100101; 526 let Inst{23-22} = opc{5-4}; 527 let Inst{21-19} = 0b010; 528 let Inst{18-16} = opc{3-1}; 529 let Inst{15-14} = 0b11; 530 let Inst{13-10} = Pg; 531 let Inst{9} = opc{0}; 532 let Inst{8-5} = Pn; 533 let Inst{4-0} = 0b00000; 534 535 let Defs = [NZCV]; 536 let isCompare = 1; 537} 538 539class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 540 PPRRegOp pprty> 541: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 542 asm, "\t$Pdn, $Pg, $_Pdn", 543 "", 544 []>, Sched<[]> { 545 bits<4> Pdn; 546 bits<4> Pg; 547 let Inst{31-24} = 0b00100101; 548 let Inst{23-22} = sz8_64; 549 let Inst{21-19} = 0b011; 550 let Inst{18-16} = opc{4-2}; 551 let Inst{15-11} = 0b11000; 552 let Inst{10-9} = opc{1-0}; 553 let Inst{8-5} = Pg; 554 let Inst{4} = 0; 555 let Inst{3-0} = Pdn; 556 557 let Constraints = "$Pdn = $_Pdn"; 558 let Defs = [NZCV]; 559} 560 561multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 562 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 563 564 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 565} 566 567multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 568 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 569 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 570 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 571 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 572 573 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 574 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 575 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 576 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 577} 578 579//===----------------------------------------------------------------------===// 580// SVE Predicate Count Group 581//===----------------------------------------------------------------------===// 582 583class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 584 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 585: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 586 asm, "\t$Rdn, $Pg", 587 "", 588 []>, Sched<[]> { 589 bits<5> Rdn; 590 bits<4> Pg; 591 let Inst{31-24} = 0b00100101; 592 let Inst{23-22} = sz8_64; 593 let Inst{21-19} = 0b101; 594 let Inst{18-16} = opc{4-2}; 595 let Inst{15-11} = 0b10001; 596 let Inst{10-9} = opc{1-0}; 597 let Inst{8-5} = Pg; 598 let Inst{4-0} = Rdn; 599 600 // Signed 32bit forms require their GPR operand printed. 601 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 602 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 603 !strconcat(asm, "\t$Rdn, $Pg")); 604 let Constraints = "$Rdn = $_Rdn"; 605} 606 607multiclass sve_int_count_r_s32<bits<5> opc, string asm, 608 SDPatternOperator op> { 609 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 610 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 611 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 612 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 613 614 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 615 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 616 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 617 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 618 619 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 620 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 621 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 622 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 623 624 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 625 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 626 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 627 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 628 629 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 630 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 631 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 632 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 633} 634 635multiclass sve_int_count_r_u32<bits<5> opc, string asm, 636 SDPatternOperator op> { 637 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 638 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 639 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 640 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 641 642 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 643 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 644 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 645 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 646 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 647 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 648 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 649 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 650} 651 652multiclass sve_int_count_r_x64<bits<5> opc, string asm, 653 SDPatternOperator op = null_frag> { 654 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 655 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 656 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 657 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 658 659 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 660 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 661 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 662 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 663 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 664 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 665 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 666 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 667} 668 669class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 670 ZPRRegOp zprty, PPRRegOp pprty> 671: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 672 asm, "\t$Zdn, $Pm", 673 "", 674 []>, Sched<[]> { 675 bits<4> Pm; 676 bits<5> Zdn; 677 let Inst{31-24} = 0b00100101; 678 let Inst{23-22} = sz8_64; 679 let Inst{21-19} = 0b101; 680 let Inst{18-16} = opc{4-2}; 681 let Inst{15-11} = 0b10000; 682 let Inst{10-9} = opc{1-0}; 683 let Inst{8-5} = Pm; 684 let Inst{4-0} = Zdn; 685 686 let Constraints = "$Zdn = $_Zdn"; 687 let DestructiveInstType = DestructiveOther; 688 let ElementSize = ElementSizeNone; 689} 690 691multiclass sve_int_count_v<bits<5> opc, string asm, 692 SDPatternOperator op = null_frag> { 693 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 694 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 695 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 696 697 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 698 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 699 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 700 701 def : InstAlias<asm # "\t$Zdn, $Pm", 702 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 703 def : InstAlias<asm # "\t$Zdn, $Pm", 704 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 705 def : InstAlias<asm # "\t$Zdn, $Pm", 706 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 707} 708 709class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 710 PPRRegOp pprty> 711: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 712 asm, "\t$Rd, $Pg, $Pn", 713 "", 714 []>, Sched<[]> { 715 bits<4> Pg; 716 bits<4> Pn; 717 bits<5> Rd; 718 let Inst{31-24} = 0b00100101; 719 let Inst{23-22} = sz8_64; 720 let Inst{21-19} = 0b100; 721 let Inst{18-16} = opc{3-1}; 722 let Inst{15-14} = 0b10; 723 let Inst{13-10} = Pg; 724 let Inst{9} = opc{0}; 725 let Inst{8-5} = Pn; 726 let Inst{4-0} = Rd; 727} 728 729multiclass sve_int_pcount_pred<bits<4> opc, string asm, 730 SDPatternOperator int_op> { 731 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 732 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 733 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 734 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 735 736 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 737 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 738 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 739 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 740} 741 742//===----------------------------------------------------------------------===// 743// SVE Element Count Group 744//===----------------------------------------------------------------------===// 745 746class sve_int_count<bits<3> opc, string asm> 747: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 748 asm, "\t$Rd, $pattern, mul $imm4", 749 "", 750 []>, Sched<[]> { 751 bits<5> Rd; 752 bits<4> imm4; 753 bits<5> pattern; 754 let Inst{31-24} = 0b00000100; 755 let Inst{23-22} = opc{2-1}; 756 let Inst{21-20} = 0b10; 757 let Inst{19-16} = imm4; 758 let Inst{15-11} = 0b11100; 759 let Inst{10} = opc{0}; 760 let Inst{9-5} = pattern; 761 let Inst{4-0} = Rd; 762} 763 764multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 765 def NAME : sve_int_count<opc, asm>; 766 767 def : InstAlias<asm # "\t$Rd, $pattern", 768 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 769 def : InstAlias<asm # "\t$Rd", 770 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 771 772 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))), 773 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 774 775 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))), 776 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 777 778 def : Pat<(i64 (op sve_pred_enum:$pattern)), 779 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 780} 781 782class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 783: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 784 asm, "\t$Zdn, $pattern, mul $imm4", 785 "", 786 []>, Sched<[]> { 787 bits<5> Zdn; 788 bits<5> pattern; 789 bits<4> imm4; 790 let Inst{31-24} = 0b00000100; 791 let Inst{23-22} = opc{4-3}; 792 let Inst{21} = 0b1; 793 let Inst{20} = opc{2}; 794 let Inst{19-16} = imm4; 795 let Inst{15-12} = 0b1100; 796 let Inst{11-10} = opc{1-0}; 797 let Inst{9-5} = pattern; 798 let Inst{4-0} = Zdn; 799 800 let Constraints = "$Zdn = $_Zdn"; 801 let DestructiveInstType = DestructiveOther; 802 let ElementSize = ElementSizeNone; 803} 804 805multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 806 SDPatternOperator op = null_frag, 807 ValueType vt = OtherVT> { 808 def NAME : sve_int_countvlv<opc, asm, zprty>; 809 810 def : InstAlias<asm # "\t$Zdn, $pattern", 811 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 812 def : InstAlias<asm # "\t$Zdn", 813 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 814 815 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 816 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 817} 818 819class sve_int_pred_pattern_a<bits<3> opc, string asm> 820: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 821 asm, "\t$Rdn, $pattern, mul $imm4", 822 "", 823 []>, Sched<[]> { 824 bits<5> Rdn; 825 bits<5> pattern; 826 bits<4> imm4; 827 let Inst{31-24} = 0b00000100; 828 let Inst{23-22} = opc{2-1}; 829 let Inst{21-20} = 0b11; 830 let Inst{19-16} = imm4; 831 let Inst{15-11} = 0b11100; 832 let Inst{10} = opc{0}; 833 let Inst{9-5} = pattern; 834 let Inst{4-0} = Rdn; 835 836 let Constraints = "$Rdn = $_Rdn"; 837} 838 839multiclass sve_int_pred_pattern_a<bits<3> opc, string asm> { 840 def NAME : sve_int_pred_pattern_a<opc, asm>; 841 842 def : InstAlias<asm # "\t$Rdn, $pattern", 843 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 844 def : InstAlias<asm # "\t$Rdn", 845 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 846} 847 848class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 849 RegisterOperand st> 850: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 851 asm, "\t$Rdn, $pattern, mul $imm4", 852 "", 853 []>, Sched<[]> { 854 bits<5> Rdn; 855 bits<5> pattern; 856 bits<4> imm4; 857 let Inst{31-24} = 0b00000100; 858 let Inst{23-22} = opc{4-3}; 859 let Inst{21} = 0b1; 860 let Inst{20} = opc{2}; 861 let Inst{19-16} = imm4; 862 let Inst{15-12} = 0b1111; 863 let Inst{11-10} = opc{1-0}; 864 let Inst{9-5} = pattern; 865 let Inst{4-0} = Rdn; 866 867 // Signed 32bit forms require their GPR operand printed. 868 let AsmString = !if(!eq(opc{2,0}, 0b00), 869 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 870 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 871 872 let Constraints = "$Rdn = $_Rdn"; 873} 874 875multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 876 SDPatternOperator op> { 877 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 878 879 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 880 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 881 def : InstAlias<asm # "\t$Rd, $Rn", 882 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 883 884 // NOTE: Register allocation doesn't like tied operands of differing register 885 // class, hence the extra INSERT_SUBREG complication. 886 887 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 888 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 889 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 890 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 891} 892 893multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 894 SDPatternOperator op> { 895 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 896 897 def : InstAlias<asm # "\t$Rdn, $pattern", 898 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 899 def : InstAlias<asm # "\t$Rdn", 900 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 901 902 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 903 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 904} 905 906multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 907 SDPatternOperator op> { 908 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 909 910 def : InstAlias<asm # "\t$Rdn, $pattern", 911 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 912 def : InstAlias<asm # "\t$Rdn", 913 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 914 915 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 916 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 917} 918 919 920//===----------------------------------------------------------------------===// 921// SVE Permute - Cross Lane Group 922//===----------------------------------------------------------------------===// 923 924class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 925 ValueType vt, RegisterClass srcRegType, 926 SDPatternOperator op> 927: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 928 asm, "\t$Zd, $Rn", 929 "", 930 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 931 bits<5> Rn; 932 bits<5> Zd; 933 let Inst{31-24} = 0b00000101; 934 let Inst{23-22} = sz8_64; 935 let Inst{21-10} = 0b100000001110; 936 let Inst{9-5} = Rn; 937 let Inst{4-0} = Zd; 938} 939 940multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 941 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 942 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 943 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 944 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 945 946 def : InstAlias<"mov $Zd, $Rn", 947 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 948 def : InstAlias<"mov $Zd, $Rn", 949 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 950 def : InstAlias<"mov $Zd, $Rn", 951 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 952 def : InstAlias<"mov $Zd, $Rn", 953 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 954} 955 956class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 957 ZPRRegOp zprty> 958: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 959 asm, "\t$Zd, $Zn$idx", 960 "", 961 []>, Sched<[]> { 962 bits<5> Zd; 963 bits<5> Zn; 964 bits<7> idx; 965 let Inst{31-24} = 0b00000101; 966 let Inst{23-22} = {?,?}; // imm3h 967 let Inst{21} = 0b1; 968 let Inst{20-16} = tsz; 969 let Inst{15-10} = 0b001000; 970 let Inst{9-5} = Zn; 971 let Inst{4-0} = Zd; 972} 973 974multiclass sve_int_perm_dup_i<string asm> { 975 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 976 let Inst{23-22} = idx{5-4}; 977 let Inst{20-17} = idx{3-0}; 978 } 979 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 980 let Inst{23-22} = idx{4-3}; 981 let Inst{20-18} = idx{2-0}; 982 } 983 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 984 let Inst{23-22} = idx{3-2}; 985 let Inst{20-19} = idx{1-0}; 986 } 987 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 988 let Inst{23-22} = idx{2-1}; 989 let Inst{20} = idx{0}; 990 } 991 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 992 let Inst{23-22} = idx{1-0}; 993 } 994 995 def : InstAlias<"mov $Zd, $Zn$idx", 996 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 997 def : InstAlias<"mov $Zd, $Zn$idx", 998 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 999 def : InstAlias<"mov $Zd, $Zn$idx", 1000 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1001 def : InstAlias<"mov $Zd, $Zn$idx", 1002 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1003 def : InstAlias<"mov $Zd, $Zn$idx", 1004 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1005 def : InstAlias<"mov $Zd, $Bn", 1006 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1007 def : InstAlias<"mov $Zd, $Hn", 1008 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1009 def : InstAlias<"mov $Zd, $Sn", 1010 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1011 def : InstAlias<"mov $Zd, $Dn", 1012 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1013 def : InstAlias<"mov $Zd, $Qn", 1014 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1015} 1016 1017class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1018 RegisterOperand VecList> 1019: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1020 asm, "\t$Zd, $Zn, $Zm", 1021 "", 1022 []>, Sched<[]> { 1023 bits<5> Zd; 1024 bits<5> Zm; 1025 bits<5> Zn; 1026 let Inst{31-24} = 0b00000101; 1027 let Inst{23-22} = sz8_64; 1028 let Inst{21} = 0b1; 1029 let Inst{20-16} = Zm; 1030 let Inst{15-13} = 0b001; 1031 let Inst{12-11} = opc; 1032 let Inst{10} = 0b0; 1033 let Inst{9-5} = Zn; 1034 let Inst{4-0} = Zd; 1035} 1036 1037multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1038 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1039 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1040 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1041 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1042 1043 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1044 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1045 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1046 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1047 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1048 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1049 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1050 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1051 1052 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1053 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1054 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1055 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1056 1057 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1058 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1059 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1060 1061 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1062} 1063 1064multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1065 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1066 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1067 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1068 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1069 1070 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1071 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1072 nxv16i8:$Op2, zsub1), 1073 nxv16i8:$Op3))>; 1074 1075 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1076 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1077 nxv8i16:$Op2, zsub1), 1078 nxv8i16:$Op3))>; 1079 1080 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1081 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1082 nxv4i32:$Op2, zsub1), 1083 nxv4i32:$Op3))>; 1084 1085 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1086 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1087 nxv2i64:$Op2, zsub1), 1088 nxv2i64:$Op3))>; 1089 1090 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1091 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1092 nxv8f16:$Op2, zsub1), 1093 nxv8i16:$Op3))>; 1094 1095 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1096 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1097 nxv4f32:$Op2, zsub1), 1098 nxv4i32:$Op3))>; 1099 1100 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1101 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1102 nxv2f64:$Op2, zsub1), 1103 nxv2i64:$Op3))>; 1104 1105 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1106 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1107 nxv8bf16:$Op2, zsub1), 1108 nxv8i16:$Op3))>; 1109} 1110 1111class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1112: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1113 asm, "\t$Zd, $Zn, $Zm", 1114 "", 1115 []>, Sched<[]> { 1116 bits<5> Zd; 1117 bits<5> Zm; 1118 bits<5> Zn; 1119 let Inst{31-24} = 0b00000101; 1120 let Inst{23-22} = sz8_64; 1121 let Inst{21} = 0b1; 1122 let Inst{20-16} = Zm; 1123 let Inst{15-10} = 0b001011; 1124 let Inst{9-5} = Zn; 1125 let Inst{4-0} = Zd; 1126 1127 let Constraints = "$Zd = $_Zd"; 1128} 1129 1130multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1131 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1132 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1133 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1134 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1135 1136 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1137 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1138 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1139 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1140 1141 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1142 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1143 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1144 1145 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1146} 1147 1148class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1149: I<(outs zprty:$Zd), (ins zprty:$Zn), 1150 asm, "\t$Zd, $Zn", 1151 "", 1152 []>, Sched<[]> { 1153 bits<5> Zd; 1154 bits<5> Zn; 1155 let Inst{31-24} = 0b00000101; 1156 let Inst{23-22} = sz8_64; 1157 let Inst{21-10} = 0b111000001110; 1158 let Inst{9-5} = Zn; 1159 let Inst{4-0} = Zd; 1160} 1161 1162multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1163 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1164 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1165 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1166 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1167 1168 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1169 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1170 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1171 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1172 1173 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1174 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1175 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1176 1177 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1178} 1179 1180class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1181: I<(outs pprty:$Pd), (ins pprty:$Pn), 1182 asm, "\t$Pd, $Pn", 1183 "", 1184 []>, Sched<[]> { 1185 bits<4> Pd; 1186 bits<4> Pn; 1187 let Inst{31-24} = 0b00000101; 1188 let Inst{23-22} = sz8_64; 1189 let Inst{21-9} = 0b1101000100000; 1190 let Inst{8-5} = Pn; 1191 let Inst{4} = 0b0; 1192 let Inst{3-0} = Pd; 1193} 1194 1195multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1196 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1197 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1198 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1199 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1200 1201 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1202 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1203 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1204 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1205} 1206 1207class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1208 ZPRRegOp zprty1, ZPRRegOp zprty2> 1209: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1210 asm, "\t$Zd, $Zn", 1211 "", []>, Sched<[]> { 1212 bits<5> Zd; 1213 bits<5> Zn; 1214 let Inst{31-24} = 0b00000101; 1215 let Inst{23-22} = sz16_64; 1216 let Inst{21-18} = 0b1100; 1217 let Inst{17-16} = opc; 1218 let Inst{15-10} = 0b001110; 1219 let Inst{9-5} = Zn; 1220 let Inst{4-0} = Zd; 1221} 1222 1223multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1224 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1225 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1226 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1227 1228 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1229 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1230 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1231} 1232 1233class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1234 RegisterClass srcRegType> 1235: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1236 asm, "\t$Zdn, $Rm", 1237 "", 1238 []>, Sched<[]> { 1239 bits<5> Rm; 1240 bits<5> Zdn; 1241 let Inst{31-24} = 0b00000101; 1242 let Inst{23-22} = sz8_64; 1243 let Inst{21-10} = 0b100100001110; 1244 let Inst{9-5} = Rm; 1245 let Inst{4-0} = Zdn; 1246 1247 let Constraints = "$Zdn = $_Zdn"; 1248 let DestructiveInstType = DestructiveOther; 1249} 1250 1251multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1252 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1253 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1254 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1255 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1256 1257 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1258 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1259 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1260 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1261} 1262 1263class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1264 RegisterClass srcRegType> 1265: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Vm), 1266 asm, "\t$Zdn, $Vm", 1267 "", 1268 []>, Sched<[]> { 1269 bits<5> Vm; 1270 bits<5> Zdn; 1271 let Inst{31-24} = 0b00000101; 1272 let Inst{23-22} = sz8_64; 1273 let Inst{21-10} = 0b110100001110; 1274 let Inst{9-5} = Vm; 1275 let Inst{4-0} = Zdn; 1276 1277 let Constraints = "$Zdn = $_Zdn"; 1278 let DestructiveInstType = DestructiveOther; 1279} 1280 1281multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1282 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8>; 1283 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16>; 1284 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32>; 1285 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64>; 1286 1287 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, f16, !cast<Instruction>(NAME # _H)>; 1288 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, f32, !cast<Instruction>(NAME # _S)>; 1289 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, f64, !cast<Instruction>(NAME # _D)>; 1290 1291 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, bf16, !cast<Instruction>(NAME # _H)>; 1292} 1293 1294//===----------------------------------------------------------------------===// 1295// SVE Permute - Extract Group 1296//===----------------------------------------------------------------------===// 1297 1298class sve_int_perm_extract_i<string asm> 1299: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1300 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1301 "", []>, Sched<[]> { 1302 bits<5> Zdn; 1303 bits<5> Zm; 1304 bits<8> imm8; 1305 let Inst{31-21} = 0b00000101001; 1306 let Inst{20-16} = imm8{7-3}; 1307 let Inst{15-13} = 0b000; 1308 let Inst{12-10} = imm8{2-0}; 1309 let Inst{9-5} = Zm; 1310 let Inst{4-0} = Zdn; 1311 1312 let Constraints = "$Zdn = $_Zdn"; 1313 let DestructiveInstType = DestructiveOther; 1314 let ElementSize = ElementSizeNone; 1315} 1316 1317multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1318 def NAME : sve_int_perm_extract_i<asm>; 1319 1320 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1321 !cast<Instruction>(NAME)>; 1322} 1323 1324class sve2_int_perm_extract_i_cons<string asm> 1325: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1326 asm, "\t$Zd, $Zn, $imm8", 1327 "", []>, Sched<[]> { 1328 bits<5> Zd; 1329 bits<5> Zn; 1330 bits<8> imm8; 1331 let Inst{31-21} = 0b00000101011; 1332 let Inst{20-16} = imm8{7-3}; 1333 let Inst{15-13} = 0b000; 1334 let Inst{12-10} = imm8{2-0}; 1335 let Inst{9-5} = Zn; 1336 let Inst{4-0} = Zd; 1337} 1338 1339//===----------------------------------------------------------------------===// 1340// SVE Vector Select Group 1341//===----------------------------------------------------------------------===// 1342 1343class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1344: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1345 asm, "\t$Zd, $Pg, $Zn, $Zm", 1346 "", 1347 []>, Sched<[]> { 1348 bits<4> Pg; 1349 bits<5> Zd; 1350 bits<5> Zm; 1351 bits<5> Zn; 1352 let Inst{31-24} = 0b00000101; 1353 let Inst{23-22} = sz8_64; 1354 let Inst{21} = 0b1; 1355 let Inst{20-16} = Zm; 1356 let Inst{15-14} = 0b11; 1357 let Inst{13-10} = Pg; 1358 let Inst{9-5} = Zn; 1359 let Inst{4-0} = Zd; 1360} 1361 1362multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1363 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1364 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1365 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1366 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1367 1368 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1369 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1370 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1371 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1372 1373 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1374 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1375 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1376 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1377 1378 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1379 1380 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1381 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1382 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1383 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1384 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1385 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1386 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1387 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1388} 1389 1390 1391//===----------------------------------------------------------------------===// 1392// SVE Predicate Logical Operations Group 1393//===----------------------------------------------------------------------===// 1394 1395class sve_int_pred_log<bits<4> opc, string asm> 1396: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1397 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1398 "", 1399 []>, Sched<[]> { 1400 bits<4> Pd; 1401 bits<4> Pg; 1402 bits<4> Pm; 1403 bits<4> Pn; 1404 let Inst{31-24} = 0b00100101; 1405 let Inst{23-22} = opc{3-2}; 1406 let Inst{21-20} = 0b00; 1407 let Inst{19-16} = Pm; 1408 let Inst{15-14} = 0b01; 1409 let Inst{13-10} = Pg; 1410 let Inst{9} = opc{1}; 1411 let Inst{8-5} = Pn; 1412 let Inst{4} = opc{0}; 1413 let Inst{3-0} = Pd; 1414 1415 // SEL has no predication qualifier. 1416 let AsmString = !if(!eq(opc, 0b0011), 1417 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1418 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1419 1420 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1421 1422} 1423 1424multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1425 SDPatternOperator op_nopred = null_frag> { 1426 def NAME : sve_int_pred_log<opc, asm>; 1427 1428 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1429 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1430 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1431 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1432 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1433 !cast<Instruction>(NAME), PTRUE_B>; 1434 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1435 !cast<Instruction>(NAME), PTRUE_H>; 1436 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1437 !cast<Instruction>(NAME), PTRUE_S>; 1438 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1439 !cast<Instruction>(NAME), PTRUE_D>; 1440} 1441 1442//===----------------------------------------------------------------------===// 1443// SVE Logical Mask Immediate Group 1444//===----------------------------------------------------------------------===// 1445 1446class sve_int_log_imm<bits<2> opc, string asm> 1447: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1448 asm, "\t$Zdn, $_Zdn, $imms13", 1449 "", []>, Sched<[]> { 1450 bits<5> Zdn; 1451 bits<13> imms13; 1452 let Inst{31-24} = 0b00000101; 1453 let Inst{23-22} = opc; 1454 let Inst{21-18} = 0b0000; 1455 let Inst{17-5} = imms13; 1456 let Inst{4-0} = Zdn; 1457 1458 let Constraints = "$Zdn = $_Zdn"; 1459 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1460 let DestructiveInstType = DestructiveOther; 1461 let ElementSize = ElementSizeNone; 1462} 1463 1464multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1465 def NAME : sve_int_log_imm<opc, asm>; 1466 1467 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1468 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1469 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1470 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1471 1472 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1473 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1474 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1475 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1476 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1477 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1478 1479 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1480 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1481 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1482 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1483 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1484 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1485 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1486 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1487} 1488 1489class sve_int_dup_mask_imm<string asm> 1490: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1491 asm, "\t$Zd, $imms", 1492 "", 1493 []>, Sched<[]> { 1494 bits<5> Zd; 1495 bits<13> imms; 1496 let Inst{31-18} = 0b00000101110000; 1497 let Inst{17-5} = imms; 1498 let Inst{4-0} = Zd; 1499 1500 let isReMaterializable = 1; 1501 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1502} 1503 1504multiclass sve_int_dup_mask_imm<string asm> { 1505 def NAME : sve_int_dup_mask_imm<asm>; 1506 1507 def : InstAlias<"dupm $Zd, $imm", 1508 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1509 def : InstAlias<"dupm $Zd, $imm", 1510 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1511 def : InstAlias<"dupm $Zd, $imm", 1512 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1513 1514 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1515 def : InstAlias<"mov $Zd, $imm", 1516 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1517 def : InstAlias<"mov $Zd, $imm", 1518 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1519 def : InstAlias<"mov $Zd, $imm", 1520 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1521} 1522 1523//===----------------------------------------------------------------------===// 1524// SVE Integer Arithmetic - Unpredicated Group. 1525//===----------------------------------------------------------------------===// 1526 1527class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1528 ZPRRegOp zprty> 1529: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1530 asm, "\t$Zd, $Zn, $Zm", 1531 "", []>, Sched<[]> { 1532 bits<5> Zd; 1533 bits<5> Zm; 1534 bits<5> Zn; 1535 let Inst{31-24} = 0b00000100; 1536 let Inst{23-22} = sz8_64; 1537 let Inst{21} = 0b1; 1538 let Inst{20-16} = Zm; 1539 let Inst{15-13} = 0b000; 1540 let Inst{12-10} = opc; 1541 let Inst{9-5} = Zn; 1542 let Inst{4-0} = Zd; 1543} 1544 1545multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, 1546 SDPatternOperator op, SDPatternOperator int_op> { 1547 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1548 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1549 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1550 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1551 1552 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1553 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1554 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1555 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1556 1557 // Intrinsic version 1558 def : SVE_2_Op_Pat<nxv16i8, int_op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1559 def : SVE_2_Op_Pat<nxv8i16, int_op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1560 def : SVE_2_Op_Pat<nxv4i32, int_op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1561 def : SVE_2_Op_Pat<nxv2i64, int_op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1562} 1563 1564//===----------------------------------------------------------------------===// 1565// SVE Floating Point Arithmetic - Predicated Group 1566//===----------------------------------------------------------------------===// 1567 1568class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1569 ZPRRegOp zprty, 1570 Operand imm_ty> 1571: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1572 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1573 "", 1574 []>, Sched<[]> { 1575 bits<3> Pg; 1576 bits<5> Zdn; 1577 bit i1; 1578 let Inst{31-24} = 0b01100101; 1579 let Inst{23-22} = sz; 1580 let Inst{21-19} = 0b011; 1581 let Inst{18-16} = opc; 1582 let Inst{15-13} = 0b100; 1583 let Inst{12-10} = Pg; 1584 let Inst{9-6} = 0b0000; 1585 let Inst{5} = i1; 1586 let Inst{4-0} = Zdn; 1587 1588 let Constraints = "$Zdn = $_Zdn"; 1589 let DestructiveInstType = DestructiveOther; 1590 let ElementSize = zprty.ElementSize; 1591} 1592 1593multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, Operand imm_ty> { 1594 def _H : sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1595 def _S : sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1596 def _D : sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1597} 1598 1599class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1600 ZPRRegOp zprty> 1601: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1602 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1603 "", 1604 []>, Sched<[]> { 1605 bits<3> Pg; 1606 bits<5> Zdn; 1607 bits<5> Zm; 1608 let Inst{31-24} = 0b01100101; 1609 let Inst{23-22} = sz; 1610 let Inst{21-20} = 0b00; 1611 let Inst{19-16} = opc; 1612 let Inst{15-13} = 0b100; 1613 let Inst{12-10} = Pg; 1614 let Inst{9-5} = Zm; 1615 let Inst{4-0} = Zdn; 1616 1617 let Constraints = "$Zdn = $_Zdn"; 1618 let DestructiveInstType = DestructiveOther; 1619 let ElementSize = zprty.ElementSize; 1620} 1621 1622multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1623 SDPatternOperator op, DestructiveInstTypeEnum flags, 1624 string revname="", bit isReverseInstr=0> { 1625 let DestructiveInstType = flags in { 1626 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1627 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1628 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1629 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1630 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1631 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1632 } 1633 1634 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1635 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1636 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1637} 1638 1639multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1640 SDPatternOperator op> { 1641 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1642 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1643 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1644 1645 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1646 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1647 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1648} 1649 1650multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1651 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1652 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1653 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1654 1655 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1656 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1657 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1658} 1659 1660class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1661: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, imm32_0_7:$imm3), 1662 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1663 "", 1664 []>, Sched<[]> { 1665 bits<5> Zdn; 1666 bits<5> Zm; 1667 bits<3> imm3; 1668 let Inst{31-24} = 0b01100101; 1669 let Inst{23-22} = sz; 1670 let Inst{21-19} = 0b010; 1671 let Inst{18-16} = imm3; 1672 let Inst{15-10} = 0b100000; 1673 let Inst{9-5} = Zm; 1674 let Inst{4-0} = Zdn; 1675 1676 let Constraints = "$Zdn = $_Zdn"; 1677 let DestructiveInstType = DestructiveOther; 1678 let ElementSize = ElementSizeNone; 1679} 1680 1681multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1682 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1683 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1684 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1685 1686 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 imm32_0_7:$imm))), 1687 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, imm32_0_7:$imm)>; 1688 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 imm32_0_7:$imm))), 1689 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, imm32_0_7:$imm)>; 1690 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 imm32_0_7:$imm))), 1691 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, imm32_0_7:$imm)>; 1692} 1693 1694//===----------------------------------------------------------------------===// 1695// SVE Floating Point Arithmetic - Unpredicated Group 1696//===----------------------------------------------------------------------===// 1697 1698class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 1699: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1700 asm, "\t$Zd, $Zn, $Zm", 1701 "", 1702 []>, Sched<[]> { 1703 bits<5> Zd; 1704 bits<5> Zm; 1705 bits<5> Zn; 1706 let Inst{31-24} = 0b01100101; 1707 let Inst{23-22} = sz; 1708 let Inst{21} = 0b0; 1709 let Inst{20-16} = Zm; 1710 let Inst{15-13} = 0b000; 1711 let Inst{12-10} = opc; 1712 let Inst{9-5} = Zn; 1713 let Inst{4-0} = Zd; 1714} 1715 1716multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 1717 SDPatternOperator predicated_op = null_frag> { 1718 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1719 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1720 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1721 1722 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1723 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1724 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1725 1726 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1727 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1728 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1729} 1730 1731multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 1732 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1733 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1734 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1735 1736 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1737 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1738 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1739} 1740 1741//===----------------------------------------------------------------------===// 1742// SVE Floating Point Fused Multiply-Add Group 1743//===----------------------------------------------------------------------===// 1744 1745class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 1746: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 1747 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 1748 "", 1749 []>, Sched<[]> { 1750 bits<3> Pg; 1751 bits<5> Zda; 1752 bits<5> Zm; 1753 bits<5> Zn; 1754 let Inst{31-24} = 0b01100101; 1755 let Inst{23-22} = sz; 1756 let Inst{21} = 0b1; 1757 let Inst{20-16} = Zm; 1758 let Inst{15} = 0b0; 1759 let Inst{14-13} = opc; 1760 let Inst{12-10} = Pg; 1761 let Inst{9-5} = Zn; 1762 let Inst{4-0} = Zda; 1763 1764 let Constraints = "$Zda = $_Zda"; 1765 let DestructiveInstType = DestructiveOther; 1766 let ElementSize = zprty.ElementSize; 1767} 1768 1769multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, SDPatternOperator op> { 1770 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>; 1771 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>; 1772 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>; 1773 1774 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1775 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1776 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1777} 1778 1779class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 1780 ZPRRegOp zprty> 1781: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 1782 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 1783 "", 1784 []>, Sched<[]> { 1785 bits<3> Pg; 1786 bits<5> Za; 1787 bits<5> Zdn; 1788 bits<5> Zm; 1789 let Inst{31-24} = 0b01100101; 1790 let Inst{23-22} = sz; 1791 let Inst{21} = 0b1; 1792 let Inst{20-16} = Za; 1793 let Inst{15} = 0b1; 1794 let Inst{14-13} = opc; 1795 let Inst{12-10} = Pg; 1796 let Inst{9-5} = Zm; 1797 let Inst{4-0} = Zdn; 1798 1799 let Constraints = "$Zdn = $_Zdn"; 1800 let DestructiveInstType = DestructiveOther; 1801 let ElementSize = zprty.ElementSize; 1802} 1803 1804multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op> { 1805 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>; 1806 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>; 1807 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>; 1808 1809 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1810 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1811 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1812} 1813 1814//===----------------------------------------------------------------------===// 1815// SVE Floating Point Multiply-Add - Indexed Group 1816//===----------------------------------------------------------------------===// 1817 1818class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 1819 ZPRRegOp zprty1, 1820 ZPRRegOp zprty2, Operand itype> 1821: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 1822 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 1823 bits<5> Zda; 1824 bits<5> Zn; 1825 let Inst{31-24} = 0b01100100; 1826 let Inst{23-22} = sz; 1827 let Inst{21} = 0b1; 1828 let Inst{15-11} = 0; 1829 let Inst{10} = opc; 1830 let Inst{9-5} = Zn; 1831 let Inst{4-0} = Zda; 1832 1833 let Constraints = "$Zda = $_Zda"; 1834 let DestructiveInstType = DestructiveOther; 1835 let ElementSize = ElementSizeNone; 1836} 1837 1838multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 1839 SDPatternOperator op> { 1840 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 1841 bits<3> Zm; 1842 bits<3> iop; 1843 let Inst{22} = iop{2}; 1844 let Inst{20-19} = iop{1-0}; 1845 let Inst{18-16} = Zm; 1846 } 1847 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 1848 bits<3> Zm; 1849 bits<2> iop; 1850 let Inst{20-19} = iop; 1851 let Inst{18-16} = Zm; 1852 } 1853 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 1854 bits<4> Zm; 1855 bit iop; 1856 let Inst{20} = iop; 1857 let Inst{19-16} = Zm; 1858 } 1859 1860 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 1861 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 1862 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 1863 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 1864 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 1865 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 1866} 1867 1868 1869//===----------------------------------------------------------------------===// 1870// SVE Floating Point Multiply - Indexed Group 1871//===----------------------------------------------------------------------===// 1872 1873class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 1874 ZPRRegOp zprty2, Operand itype> 1875: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 1876 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 1877 bits<5> Zd; 1878 bits<5> Zn; 1879 let Inst{31-24} = 0b01100100; 1880 let Inst{23-22} = sz; 1881 let Inst{21} = 0b1; 1882 let Inst{15-10} = 0b001000; 1883 let Inst{9-5} = Zn; 1884 let Inst{4-0} = Zd; 1885} 1886 1887multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 1888 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 1889 bits<3> Zm; 1890 bits<3> iop; 1891 let Inst{22} = iop{2}; 1892 let Inst{20-19} = iop{1-0}; 1893 let Inst{18-16} = Zm; 1894 } 1895 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 1896 bits<3> Zm; 1897 bits<2> iop; 1898 let Inst{20-19} = iop; 1899 let Inst{18-16} = Zm; 1900 } 1901 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 1902 bits<4> Zm; 1903 bit iop; 1904 let Inst{20} = iop; 1905 let Inst{19-16} = Zm; 1906 } 1907 1908 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 1909 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 1910 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 1911 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 1912 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 1913 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 1914} 1915 1916//===----------------------------------------------------------------------===// 1917// SVE Floating Point Complex Multiply-Add Group 1918//===----------------------------------------------------------------------===// 1919 1920class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 1921: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 1922 complexrotateop:$imm), 1923 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 1924 "", []>, Sched<[]> { 1925 bits<5> Zda; 1926 bits<3> Pg; 1927 bits<5> Zn; 1928 bits<5> Zm; 1929 bits<2> imm; 1930 let Inst{31-24} = 0b01100100; 1931 let Inst{23-22} = sz; 1932 let Inst{21} = 0; 1933 let Inst{20-16} = Zm; 1934 let Inst{15} = 0; 1935 let Inst{14-13} = imm; 1936 let Inst{12-10} = Pg; 1937 let Inst{9-5} = Zn; 1938 let Inst{4-0} = Zda; 1939 1940 let Constraints = "$Zda = $_Zda"; 1941 let DestructiveInstType = DestructiveOther; 1942 let ElementSize = zprty.ElementSize; 1943} 1944 1945multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 1946 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 1947 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 1948 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 1949 1950 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 1951 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 1952 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 1953 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 1954 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 1955 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 1956} 1957 1958//===----------------------------------------------------------------------===// 1959// SVE Floating Point Complex Multiply-Add - Indexed Group 1960//===----------------------------------------------------------------------===// 1961 1962class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 1963 ZPRRegOp zprty, 1964 ZPRRegOp zprty2, Operand itype> 1965: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 1966 complexrotateop:$imm), 1967 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 1968 "", []>, Sched<[]> { 1969 bits<5> Zda; 1970 bits<5> Zn; 1971 bits<2> imm; 1972 let Inst{31-24} = 0b01100100; 1973 let Inst{23-22} = sz; 1974 let Inst{21} = 0b1; 1975 let Inst{15-12} = 0b0001; 1976 let Inst{11-10} = imm; 1977 let Inst{9-5} = Zn; 1978 let Inst{4-0} = Zda; 1979 1980 let Constraints = "$Zda = $_Zda"; 1981 let DestructiveInstType = DestructiveOther; 1982 let ElementSize = ElementSizeNone; 1983} 1984 1985multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 1986 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 1987 bits<3> Zm; 1988 bits<2> iop; 1989 let Inst{20-19} = iop; 1990 let Inst{18-16} = Zm; 1991 } 1992 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 1993 bits<4> Zm; 1994 bits<1> iop; 1995 let Inst{20} = iop; 1996 let Inst{19-16} = Zm; 1997 } 1998 1999 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2000 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2001 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2002 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2003} 2004 2005//===----------------------------------------------------------------------===// 2006// SVE Floating Point Complex Addition Group 2007//===----------------------------------------------------------------------===// 2008 2009class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2010: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2011 complexrotateopodd:$imm), 2012 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2013 "", 2014 []>, Sched<[]> { 2015 bits<5> Zdn; 2016 bits<5> Zm; 2017 bits<3> Pg; 2018 bit imm; 2019 let Inst{31-24} = 0b01100100; 2020 let Inst{23-22} = sz; 2021 let Inst{21-17} = 0; 2022 let Inst{16} = imm; 2023 let Inst{15-13} = 0b100; 2024 let Inst{12-10} = Pg; 2025 let Inst{9-5} = Zm; 2026 let Inst{4-0} = Zdn; 2027 2028 let Constraints = "$Zdn = $_Zdn"; 2029 let DestructiveInstType = DestructiveOther; 2030 let ElementSize = zprty.ElementSize; 2031} 2032 2033multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2034 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2035 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2036 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2037 2038 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2039 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2040 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2041 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2042 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2043 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2044} 2045 2046//===----------------------------------------------------------------------===// 2047// SVE2 Floating Point Convert Group 2048//===----------------------------------------------------------------------===// 2049 2050class sve2_fp_convert_precision<bits<4> opc, string asm, 2051 ZPRRegOp zprty1, ZPRRegOp zprty2> 2052: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2053 asm, "\t$Zd, $Pg/m, $Zn", 2054 "", 2055 []>, Sched<[]> { 2056 bits<5> Zd; 2057 bits<5> Zn; 2058 bits<3> Pg; 2059 let Inst{31-24} = 0b01100100; 2060 let Inst{23-22} = opc{3-2}; 2061 let Inst{21-18} = 0b0010; 2062 let Inst{17-16} = opc{1-0}; 2063 let Inst{15-13} = 0b101; 2064 let Inst{12-10} = Pg; 2065 let Inst{9-5} = Zn; 2066 let Inst{4-0} = Zd; 2067 2068 let Constraints = "$Zd = $_Zd"; 2069} 2070 2071multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2072 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2073 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2074 2075 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2076 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2077} 2078 2079multiclass sve2_fp_convert_up_long<string asm, string op> { 2080 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2081 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2082 2083 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2084 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2085} 2086 2087multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2088 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2089 2090 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2091} 2092 2093//===----------------------------------------------------------------------===// 2094// SVE2 Floating Point Pairwise Group 2095//===----------------------------------------------------------------------===// 2096 2097class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2098 ZPRRegOp zprty> 2099: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2100 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2101 "", 2102 []>, Sched<[]> { 2103 bits<3> Pg; 2104 bits<5> Zm; 2105 bits<5> Zdn; 2106 let Inst{31-24} = 0b01100100; 2107 let Inst{23-22} = sz; 2108 let Inst{21-19} = 0b010; 2109 let Inst{18-16} = opc; 2110 let Inst{15-13} = 0b100; 2111 let Inst{12-10} = Pg; 2112 let Inst{9-5} = Zm; 2113 let Inst{4-0} = Zdn; 2114 2115 let Constraints = "$Zdn = $_Zdn"; 2116 let DestructiveInstType = DestructiveOther; 2117 let ElementSize = zprty.ElementSize; 2118} 2119 2120multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2121 SDPatternOperator op> { 2122 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2123 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2124 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2125 2126 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2127 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2128 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2129} 2130 2131//===----------------------------------------------------------------------===// 2132// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2133//===----------------------------------------------------------------------===// 2134 2135class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2136: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2137 VectorIndexH32b:$iop), 2138 asm, "\t$Zda, $Zn, $Zm$iop", 2139 "", 2140 []>, Sched<[]> { 2141 bits<5> Zda; 2142 bits<5> Zn; 2143 bits<3> Zm; 2144 bits<3> iop; 2145 let Inst{31-21} = 0b01100100101; 2146 let Inst{20-19} = iop{2-1}; 2147 let Inst{18-16} = Zm; 2148 let Inst{15-14} = 0b01; 2149 let Inst{13} = opc{1}; 2150 let Inst{12} = 0b0; 2151 let Inst{11} = iop{0}; 2152 let Inst{10} = opc{0}; 2153 let Inst{9-5} = Zn; 2154 let Inst{4-0} = Zda; 2155 2156 let Constraints = "$Zda = $_Zda"; 2157 let DestructiveInstType = DestructiveOther; 2158 let ElementSize = ElementSizeNone; 2159} 2160 2161multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2162 SDPatternOperator op> { 2163 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2164 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2165} 2166 2167//===----------------------------------------------------------------------===// 2168// SVE2 Floating Point Widening Multiply-Add Group 2169//===----------------------------------------------------------------------===// 2170 2171class sve2_fp_mla_long<bits<2> opc, string asm> 2172: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2173 asm, "\t$Zda, $Zn, $Zm", 2174 "", 2175 []>, Sched<[]> { 2176 bits<5> Zda; 2177 bits<5> Zn; 2178 bits<5> Zm; 2179 let Inst{31-21} = 0b01100100101; 2180 let Inst{20-16} = Zm; 2181 let Inst{15-14} = 0b10; 2182 let Inst{13} = opc{1}; 2183 let Inst{12-11} = 0b00; 2184 let Inst{10} = opc{0}; 2185 let Inst{9-5} = Zn; 2186 let Inst{4-0} = Zda; 2187 2188 let Constraints = "$Zda = $_Zda"; 2189 let DestructiveInstType = DestructiveOther; 2190 let ElementSize = ElementSizeNone; 2191} 2192 2193multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2194 def NAME : sve2_fp_mla_long<opc, asm>; 2195 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2196} 2197 2198//===----------------------------------------------------------------------===// 2199// SVE Stack Allocation Group 2200//===----------------------------------------------------------------------===// 2201 2202class sve_int_arith_vl<bit opc, string asm> 2203: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2204 asm, "\t$Rd, $Rn, $imm6", 2205 "", 2206 []>, Sched<[]> { 2207 bits<5> Rd; 2208 bits<5> Rn; 2209 bits<6> imm6; 2210 let Inst{31-23} = 0b000001000; 2211 let Inst{22} = opc; 2212 let Inst{21} = 0b1; 2213 let Inst{20-16} = Rn; 2214 let Inst{15-11} = 0b01010; 2215 let Inst{10-5} = imm6; 2216 let Inst{4-0} = Rd; 2217} 2218 2219class sve_int_read_vl_a<bit op, bits<5> opc2, string asm> 2220: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2221 asm, "\t$Rd, $imm6", 2222 "", 2223 []>, Sched<[]> { 2224 bits<5> Rd; 2225 bits<6> imm6; 2226 let Inst{31-23} = 0b000001001; 2227 let Inst{22} = op; 2228 let Inst{21} = 0b1; 2229 let Inst{20-16} = opc2{4-0}; 2230 let Inst{15-11} = 0b01010; 2231 let Inst{10-5} = imm6; 2232 let Inst{4-0} = Rd; 2233} 2234 2235//===----------------------------------------------------------------------===// 2236// SVE Permute - In Lane Group 2237//===----------------------------------------------------------------------===// 2238 2239class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2240 ZPRRegOp zprty> 2241: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2242 asm, "\t$Zd, $Zn, $Zm", 2243 "", 2244 []>, Sched<[]> { 2245 bits<5> Zd; 2246 bits<5> Zm; 2247 bits<5> Zn; 2248 let Inst{31-24} = 0b00000101; 2249 let Inst{23-22} = sz8_64; 2250 let Inst{21} = 0b1; 2251 let Inst{20-16} = Zm; 2252 let Inst{15-13} = 0b011; 2253 let Inst{12-10} = opc; 2254 let Inst{9-5} = Zn; 2255 let Inst{4-0} = Zd; 2256} 2257 2258multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2259 SDPatternOperator op> { 2260 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2261 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2262 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2263 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2264 2265 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2266 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2267 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2268 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2269 2270 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2271 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2272 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2273 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2274 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2275 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2276 2277 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2278} 2279 2280//===----------------------------------------------------------------------===// 2281// SVE Floating Point Unary Operations Group 2282//===----------------------------------------------------------------------===// 2283 2284class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2285 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2286: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2287 asm, "\t$Zd, $Pg/m, $Zn", 2288 "", 2289 []>, Sched<[]> { 2290 bits<3> Pg; 2291 bits<5> Zd; 2292 bits<5> Zn; 2293 let Inst{31-24} = 0b01100101; 2294 let Inst{23-22} = opc{6-5}; 2295 let Inst{21} = 0b0; 2296 let Inst{20-16} = opc{4-0}; 2297 let Inst{15-13} = 0b101; 2298 let Inst{12-10} = Pg; 2299 let Inst{9-5} = Zn; 2300 let Inst{4-0} = Zd; 2301 2302 let Constraints = "$Zd = $_Zd"; 2303 let DestructiveInstType = DestructiveOther; 2304 let ElementSize = Sz; 2305} 2306 2307multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2308 RegisterOperand i_zprtype, 2309 RegisterOperand o_zprtype, 2310 SDPatternOperator int_op, 2311 SDPatternOperator ir_op, ValueType vt1, 2312 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2313 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2314 2315 // convert vt1 to a packed type for the intrinsic patterns 2316 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2317 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2318 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2319 1 : vt1); 2320 2321 // convert vt3 to a packed type for the intrinsic patterns 2322 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2323 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2324 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2325 1 : vt3); 2326 2327 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2328 2329 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2330} 2331 2332multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2333 RegisterOperand i_zprtype, 2334 RegisterOperand o_zprtype, 2335 SDPatternOperator int_op, 2336 SDPatternOperator ir_op, ValueType vt1, 2337 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2338 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2339 2340 // convert vt1 to a packed type for the intrinsic patterns 2341 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2342 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2343 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2344 1 : vt1); 2345 2346 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2347 2348 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2349} 2350 2351multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2352 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>; 2353 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>; 2354 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>; 2355 2356 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2357 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2358 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2359 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2360 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2361 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2362} 2363 2364multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2365 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2366 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2367 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2368 2369 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2370 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2371 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2372} 2373 2374multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2375 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2376 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2377} 2378 2379//===----------------------------------------------------------------------===// 2380// SVE Floating Point Unary Operations - Unpredicated Group 2381//===----------------------------------------------------------------------===// 2382 2383class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2384 ZPRRegOp zprty> 2385: I<(outs zprty:$Zd), (ins zprty:$Zn), 2386 asm, "\t$Zd, $Zn", 2387 "", 2388 []>, Sched<[]> { 2389 bits<5> Zd; 2390 bits<5> Zn; 2391 let Inst{31-24} = 0b01100101; 2392 let Inst{23-22} = sz; 2393 let Inst{21-19} = 0b001; 2394 let Inst{18-16} = opc; 2395 let Inst{15-10} = 0b001100; 2396 let Inst{9-5} = Zn; 2397 let Inst{4-0} = Zd; 2398} 2399 2400multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2401 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2402 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2403 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2404 2405 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2406 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2407 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2408} 2409 2410//===----------------------------------------------------------------------===// 2411// SVE Integer Arithmetic - Binary Predicated Group 2412//===----------------------------------------------------------------------===// 2413 2414class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2415 string asm, ZPRRegOp zprty> 2416: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2417 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2418 bits<3> Pg; 2419 bits<5> Zdn; 2420 bits<5> Zm; 2421 let Inst{31-24} = 0b00000100; 2422 let Inst{23-22} = sz8_64; 2423 let Inst{21} = 0b0; 2424 let Inst{20-19} = fmt; 2425 let Inst{18-16} = opc; 2426 let Inst{15-13} = 0b000; 2427 let Inst{12-10} = Pg; 2428 let Inst{9-5} = Zm; 2429 let Inst{4-0} = Zdn; 2430 2431 let Constraints = "$Zdn = $_Zdn"; 2432 let DestructiveInstType = DestructiveOther; 2433 let ElementSize = zprty.ElementSize; 2434} 2435 2436multiclass sve_int_bin_pred_log<bits<3> opc, string asm, SDPatternOperator op> { 2437 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>; 2438 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>; 2439 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>; 2440 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>; 2441 2442 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2443 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2444 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2445 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2446} 2447 2448multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2449 SDPatternOperator op, 2450 DestructiveInstTypeEnum flags, 2451 string revname="", bit isReverseInstr=0> { 2452 let DestructiveInstType = flags in { 2453 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2454 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2455 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2456 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2457 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2458 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2459 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2460 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2461 } 2462 2463 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2464 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2465 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2466 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2467} 2468 2469multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2470 SDPatternOperator op, 2471 DestructiveInstTypeEnum flags> { 2472 let DestructiveInstType = flags in { 2473 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2474 SVEPseudo2Instr<Ps # _B, 1>; 2475 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2476 SVEPseudo2Instr<Ps # _H, 1>; 2477 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2478 SVEPseudo2Instr<Ps # _S, 1>; 2479 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2480 SVEPseudo2Instr<Ps # _D, 1>; 2481 } 2482 2483 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2484 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2485 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2486 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2487} 2488 2489multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2490 SDPatternOperator op, 2491 DestructiveInstTypeEnum flags> { 2492 let DestructiveInstType = flags in { 2493 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2494 SVEPseudo2Instr<Ps # _B, 1>; 2495 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2496 SVEPseudo2Instr<Ps # _H, 1>; 2497 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2498 SVEPseudo2Instr<Ps # _S, 1>; 2499 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2500 SVEPseudo2Instr<Ps # _D, 1>; 2501 } 2502 2503 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2504 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2505 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2506 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2507} 2508 2509// Special case for divides which are not defined for 8b/16b elements. 2510multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2511 SDPatternOperator op, 2512 DestructiveInstTypeEnum flags, 2513 string revname="", bit isReverseInstr=0> { 2514 let DestructiveInstType = flags in { 2515 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2516 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2517 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2518 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2519 } 2520 2521 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2522 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2523} 2524 2525//===----------------------------------------------------------------------===// 2526// SVE Integer Multiply-Add Group 2527//===----------------------------------------------------------------------===// 2528 2529class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2530 ZPRRegOp zprty> 2531: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2532 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2533 "", 2534 []>, Sched<[]> { 2535 bits<3> Pg; 2536 bits<5> Zdn; 2537 bits<5> Za; 2538 bits<5> Zm; 2539 let Inst{31-24} = 0b00000100; 2540 let Inst{23-22} = sz8_64; 2541 let Inst{21} = 0b0; 2542 let Inst{20-16} = Zm; 2543 let Inst{15-14} = 0b11; 2544 let Inst{13} = opc; 2545 let Inst{12-10} = Pg; 2546 let Inst{9-5} = Za; 2547 let Inst{4-0} = Zdn; 2548 2549 let Constraints = "$Zdn = $_Zdn"; 2550 let DestructiveInstType = DestructiveOther; 2551 let ElementSize = zprty.ElementSize; 2552} 2553 2554multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2555 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2556 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2557 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2558 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2559 2560 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2561 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2562 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2563 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2564} 2565 2566class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2567 ZPRRegOp zprty> 2568: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2569 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2570 "", 2571 []>, Sched<[]> { 2572 bits<3> Pg; 2573 bits<5> Zda; 2574 bits<5> Zm; 2575 bits<5> Zn; 2576 let Inst{31-24} = 0b00000100; 2577 let Inst{23-22} = sz8_64; 2578 let Inst{21} = 0b0; 2579 let Inst{20-16} = Zm; 2580 let Inst{15-14} = 0b01; 2581 let Inst{13} = opc; 2582 let Inst{12-10} = Pg; 2583 let Inst{9-5} = Zn; 2584 let Inst{4-0} = Zda; 2585 2586 let Constraints = "$Zda = $_Zda"; 2587 let DestructiveInstType = DestructiveOther; 2588 let ElementSize = zprty.ElementSize; 2589} 2590 2591multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op, 2592 SDPatternOperator outerop, SDPatternOperator mulop> { 2593 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2594 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2595 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2596 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2597 2598 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2599 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2600 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2601 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2602 2603 def : Pat<(outerop nxv16i8:$Op1, (mulop nxv16i1:$pred, nxv16i8:$Op2, nxv16i8:$Op3)), 2604 (!cast<Instruction>(NAME # _B) $pred, $Op1, $Op2, $Op3)>; 2605 def : Pat<(outerop nxv8i16:$Op1, (mulop nxv8i1:$pred, nxv8i16:$Op2, nxv8i16:$Op3)), 2606 (!cast<Instruction>(NAME # _H) $pred, $Op1, $Op2, $Op3)>; 2607 def : Pat<(outerop nxv4i32:$Op1, (mulop nxv4i1:$pred, nxv4i32:$Op2, nxv4i32:$Op3)), 2608 (!cast<Instruction>(NAME # _S) $pred, $Op1, $Op2, $Op3)>; 2609 def : Pat<(outerop nxv2i64:$Op1, (mulop nxv2i1:$pred, nxv2i64:$Op2, nxv2i64:$Op3)), 2610 (!cast<Instruction>(NAME # _D) $pred, $Op1, $Op2, $Op3)>; 2611} 2612 2613//===----------------------------------------------------------------------===// 2614// SVE2 Integer Multiply-Add - Unpredicated Group 2615//===----------------------------------------------------------------------===// 2616 2617class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2618 ZPRRegOp zprty1, ZPRRegOp zprty2> 2619: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2620 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2621 bits<5> Zda; 2622 bits<5> Zn; 2623 bits<5> Zm; 2624 let Inst{31-24} = 0b01000100; 2625 let Inst{23-22} = sz; 2626 let Inst{21} = 0b0; 2627 let Inst{20-16} = Zm; 2628 let Inst{15} = 0b0; 2629 let Inst{14-10} = opc; 2630 let Inst{9-5} = Zn; 2631 let Inst{4-0} = Zda; 2632 2633 let Constraints = "$Zda = $_Zda"; 2634 let DestructiveInstType = DestructiveOther; 2635 let ElementSize = ElementSizeNone; 2636} 2637 2638multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 2639 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 2640 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 2641 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 2642 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 2643 2644 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2645 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2646 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2647 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2648} 2649 2650multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 2651 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 2652 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 2653 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 2654 2655 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 2656 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 2657 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 2658} 2659 2660//===----------------------------------------------------------------------===// 2661// SVE2 Integer Multiply-Add - Indexed Group 2662//===----------------------------------------------------------------------===// 2663 2664class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 2665 ZPRRegOp zprty1, ZPRRegOp zprty2, 2666 ZPRRegOp zprty3, Operand itype> 2667: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2668 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2669 bits<5> Zda; 2670 bits<5> Zn; 2671 let Inst{31-24} = 0b01000100; 2672 let Inst{23-22} = sz; 2673 let Inst{21} = 0b1; 2674 let Inst{15-10} = opc; 2675 let Inst{9-5} = Zn; 2676 let Inst{4-0} = Zda; 2677 2678 let Constraints = "$Zda = $_Zda"; 2679 let DestructiveInstType = DestructiveOther; 2680 let ElementSize = ElementSizeNone; 2681} 2682 2683multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 2684 SDPatternOperator op> { 2685 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 2686 bits<3> Zm; 2687 bits<3> iop; 2688 let Inst{22} = iop{2}; 2689 let Inst{20-19} = iop{1-0}; 2690 let Inst{18-16} = Zm; 2691 } 2692 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 2693 bits<3> Zm; 2694 bits<2> iop; 2695 let Inst{20-19} = iop; 2696 let Inst{18-16} = Zm; 2697 } 2698 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 2699 bits<4> Zm; 2700 bit iop; 2701 let Inst{20} = iop; 2702 let Inst{19-16} = Zm; 2703 } 2704 2705 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 2706 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 2707 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 2708} 2709 2710//===----------------------------------------------------------------------===// 2711// SVE2 Integer Multiply-Add Long - Indexed Group 2712//===----------------------------------------------------------------------===// 2713 2714multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 2715 SDPatternOperator op> { 2716 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2717 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 2718 bits<3> Zm; 2719 bits<3> iop; 2720 let Inst{20-19} = iop{2-1}; 2721 let Inst{18-16} = Zm; 2722 let Inst{11} = iop{0}; 2723 } 2724 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2725 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 2726 bits<4> Zm; 2727 bits<2> iop; 2728 let Inst{20} = iop{1}; 2729 let Inst{19-16} = Zm; 2730 let Inst{11} = iop{0}; 2731 } 2732 2733 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 2734 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 2735} 2736 2737//===----------------------------------------------------------------------===// 2738// SVE Integer Dot Product Group 2739//===----------------------------------------------------------------------===// 2740 2741class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 2742 ZPRRegOp zprty2> 2743: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 2744 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2745 bits<5> Zda; 2746 bits<5> Zn; 2747 bits<5> Zm; 2748 let Inst{31-23} = 0b010001001; 2749 let Inst{22} = sz; 2750 let Inst{21} = 0; 2751 let Inst{20-16} = Zm; 2752 let Inst{15-11} = 0; 2753 let Inst{10} = U; 2754 let Inst{9-5} = Zn; 2755 let Inst{4-0} = Zda; 2756 2757 let Constraints = "$Zda = $_Zda"; 2758 let DestructiveInstType = DestructiveOther; 2759} 2760 2761multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 2762 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 2763 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 2764 2765 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 2766 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 2767} 2768 2769//===----------------------------------------------------------------------===// 2770// SVE Integer Dot Product Group - Indexed Group 2771//===----------------------------------------------------------------------===// 2772 2773class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 2774 ZPRRegOp zprty1, ZPRRegOp zprty2, 2775 ZPRRegOp zprty3, Operand itype> 2776: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2777 asm, "\t$Zda, $Zn, $Zm$iop", 2778 "", []>, Sched<[]> { 2779 bits<5> Zda; 2780 bits<5> Zn; 2781 let Inst{31-23} = 0b010001001; 2782 let Inst{22} = sz; 2783 let Inst{21} = 0b1; 2784 let Inst{15-11} = 0; 2785 let Inst{10} = U; 2786 let Inst{9-5} = Zn; 2787 let Inst{4-0} = Zda; 2788 2789 let Constraints = "$Zda = $_Zda"; 2790 let DestructiveInstType = DestructiveOther; 2791} 2792 2793multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 2794 SDPatternOperator op> { 2795 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 2796 bits<2> iop; 2797 bits<3> Zm; 2798 let Inst{20-19} = iop; 2799 let Inst{18-16} = Zm; 2800 } 2801 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 2802 bits<1> iop; 2803 bits<4> Zm; 2804 let Inst{20} = iop; 2805 let Inst{19-16} = Zm; 2806 } 2807 2808 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv16i8:$Op2, nxv16i8:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2809 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2810 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv8i16:$Op2, nxv8i16:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2811 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2812} 2813 2814//===----------------------------------------------------------------------===// 2815// SVE2 Complex Integer Dot Product Group 2816//===----------------------------------------------------------------------===// 2817 2818class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 2819 ZPRRegOp zprty1, ZPRRegOp zprty2> 2820: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 2821 complexrotateop:$rot), 2822 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 2823 bits<5> Zda; 2824 bits<5> Zn; 2825 bits<5> Zm; 2826 bits<2> rot; 2827 let Inst{31-24} = 0b01000100; 2828 let Inst{23-22} = sz; 2829 let Inst{21} = 0b0; 2830 let Inst{20-16} = Zm; 2831 let Inst{15-12} = opc; 2832 let Inst{11-10} = rot; 2833 let Inst{9-5} = Zn; 2834 let Inst{4-0} = Zda; 2835 2836 let Constraints = "$Zda = $_Zda"; 2837 let DestructiveInstType = DestructiveOther; 2838 let ElementSize = ElementSizeNone; 2839} 2840 2841multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 2842 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 2843 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 2844 2845 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 2846 (i32 complexrotateop:$imm))), 2847 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 2848 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2849 (i32 complexrotateop:$imm))), 2850 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 2851} 2852 2853//===----------------------------------------------------------------------===// 2854// SVE2 Complex Multiply-Add Group 2855//===----------------------------------------------------------------------===// 2856 2857multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 2858 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 2859 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 2860 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 2861 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 2862 2863 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 2864 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 2865 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 2866 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 2867} 2868 2869//===----------------------------------------------------------------------===// 2870// SVE2 Complex Integer Dot Product - Indexed Group 2871//===----------------------------------------------------------------------===// 2872 2873class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 2874 ZPRRegOp zprty1, ZPRRegOp zprty2, 2875 ZPRRegOp zprty3, Operand itype> 2876: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 2877 complexrotateop:$rot), 2878 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 2879 bits<5> Zda; 2880 bits<5> Zn; 2881 bits<2> rot; 2882 let Inst{31-24} = 0b01000100; 2883 let Inst{23-22} = sz; 2884 let Inst{21} = 0b1; 2885 let Inst{15-12} = opc; 2886 let Inst{11-10} = rot; 2887 let Inst{9-5} = Zn; 2888 let Inst{4-0} = Zda; 2889 2890 let Constraints = "$Zda = $_Zda"; 2891 let DestructiveInstType = DestructiveOther; 2892 let ElementSize = ElementSizeNone; 2893} 2894 2895multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 2896 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 2897 bits<2> iop; 2898 bits<3> Zm; 2899 let Inst{20-19} = iop; 2900 let Inst{18-16} = Zm; 2901 } 2902 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 2903 bit iop; 2904 bits<4> Zm; 2905 let Inst{20} = iop; 2906 let Inst{19-16} = Zm; 2907 } 2908 2909 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 2910 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2911 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2912 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2913 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2914 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2915} 2916 2917//===----------------------------------------------------------------------===// 2918// SVE2 Complex Multiply-Add - Indexed Group 2919//===----------------------------------------------------------------------===// 2920 2921multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 2922 SDPatternOperator op> { 2923 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 2924 bits<2> iop; 2925 bits<3> Zm; 2926 let Inst{20-19} = iop; 2927 let Inst{18-16} = Zm; 2928 } 2929 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 2930 bit iop; 2931 bits<4> Zm; 2932 let Inst{20} = iop; 2933 let Inst{19-16} = Zm; 2934 } 2935 2936 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2937 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2938 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2939 2940 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 2941 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2942 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2943} 2944 2945//===----------------------------------------------------------------------===// 2946// SVE2 Integer Multiply - Unpredicated Group 2947//===----------------------------------------------------------------------===// 2948 2949class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 2950: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2951 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 2952 bits<5> Zd; 2953 bits<5> Zm; 2954 bits<5> Zn; 2955 let Inst{31-24} = 0b00000100; 2956 let Inst{23-22} = sz; 2957 let Inst{21} = 0b1; 2958 let Inst{20-16} = Zm; 2959 let Inst{15-13} = 0b011; 2960 let Inst{12-10} = opc; 2961 let Inst{9-5} = Zn; 2962 let Inst{4-0} = Zd; 2963} 2964 2965multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 2966 SDPatternOperator op_pred = null_frag> { 2967 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 2968 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 2969 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 2970 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 2971 2972 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2973 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2974 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2975 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2976 2977 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2978 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2979 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2980 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2981} 2982 2983multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 2984 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 2985 2986 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2987} 2988 2989//===----------------------------------------------------------------------===// 2990// SVE2 Integer Multiply - Indexed Group 2991//===----------------------------------------------------------------------===// 2992 2993class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 2994 ZPRRegOp zprty1, ZPRRegOp zprty2, 2995 ZPRRegOp zprty3, Operand itype> 2996: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 2997 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2998 bits<5> Zd; 2999 bits<5> Zn; 3000 let Inst{31-24} = 0b01000100; 3001 let Inst{23-22} = sz; 3002 let Inst{21} = 0b1; 3003 let Inst{15-14} = 0b11; 3004 let Inst{13-10} = opc; 3005 let Inst{9-5} = Zn; 3006 let Inst{4-0} = Zd; 3007} 3008 3009multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3010 SDPatternOperator op> { 3011 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3012 bits<3> Zm; 3013 bits<3> iop; 3014 let Inst{22} = iop{2}; 3015 let Inst{20-19} = iop{1-0}; 3016 let Inst{18-16} = Zm; 3017 } 3018 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3019 bits<3> Zm; 3020 bits<2> iop; 3021 let Inst{20-19} = iop; 3022 let Inst{18-16} = Zm; 3023 } 3024 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3025 bits<4> Zm; 3026 bit iop; 3027 let Inst{20} = iop; 3028 let Inst{19-16} = Zm; 3029 } 3030 3031 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3032 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3033 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3034} 3035 3036multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3037 SDPatternOperator op> { 3038 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3039 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3040 bits<3> Zm; 3041 bits<3> iop; 3042 let Inst{20-19} = iop{2-1}; 3043 let Inst{18-16} = Zm; 3044 let Inst{11} = iop{0}; 3045 } 3046 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3047 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3048 bits<4> Zm; 3049 bits<2> iop; 3050 let Inst{20} = iop{1}; 3051 let Inst{19-16} = Zm; 3052 let Inst{11} = iop{0}; 3053 } 3054 3055 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3056 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3057} 3058 3059//===----------------------------------------------------------------------===// 3060// SVE2 Integer - Predicated Group 3061//===----------------------------------------------------------------------===// 3062 3063class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3064 ZPRRegOp zprty> 3065: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3066 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3067 bits<3> Pg; 3068 bits<5> Zm; 3069 bits<5> Zdn; 3070 let Inst{31-24} = 0b01000100; 3071 let Inst{23-22} = sz; 3072 let Inst{21-20} = 0b01; 3073 let Inst{20-16} = opc{5-1}; 3074 let Inst{15-14} = 0b10; 3075 let Inst{13} = opc{0}; 3076 let Inst{12-10} = Pg; 3077 let Inst{9-5} = Zm; 3078 let Inst{4-0} = Zdn; 3079 3080 let Constraints = "$Zdn = $_Zdn"; 3081 let DestructiveInstType = DestructiveOther; 3082 let ElementSize = zprty.ElementSize; 3083} 3084 3085multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op> { 3086 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>; 3087 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>; 3088 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>; 3089 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>; 3090 3091 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3092 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3093 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3094 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3095} 3096 3097class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3098 ZPRRegOp zprty1, ZPRRegOp zprty2> 3099: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3100 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3101 bits<3> Pg; 3102 bits<5> Zn; 3103 bits<5> Zda; 3104 let Inst{31-24} = 0b01000100; 3105 let Inst{23-22} = sz; 3106 let Inst{21-17} = 0b00010; 3107 let Inst{16} = U; 3108 let Inst{15-13} = 0b101; 3109 let Inst{12-10} = Pg; 3110 let Inst{9-5} = Zn; 3111 let Inst{4-0} = Zda; 3112 3113 let Constraints = "$Zda = $_Zda"; 3114 let DestructiveInstType = DestructiveOther; 3115 let ElementSize = zprty1.ElementSize; 3116} 3117 3118multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3119 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3120 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3121 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3122 3123 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3124 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3125 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3126} 3127 3128class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3129 string asm, ZPRRegOp zprty> 3130: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3131 asm, "\t$Zd, $Pg/m, $Zn", 3132 "", 3133 []>, Sched<[]> { 3134 bits<3> Pg; 3135 bits<5> Zd; 3136 bits<5> Zn; 3137 let Inst{31-24} = 0b01000100; 3138 let Inst{23-22} = sz; 3139 let Inst{21-20} = 0b00; 3140 let Inst{19} = Q; 3141 let Inst{18} = 0b0; 3142 let Inst{17-16} = opc; 3143 let Inst{15-13} = 0b101; 3144 let Inst{12-10} = Pg; 3145 let Inst{9-5} = Zn; 3146 let Inst{4-0} = Zd; 3147 3148 let Constraints = "$Zd = $_Zd"; 3149 let DestructiveInstType = DestructiveOther; 3150 let ElementSize = zprty.ElementSize; 3151} 3152 3153multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3154 SDPatternOperator op> { 3155 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>; 3156 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3157} 3158 3159multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3160 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>; 3161 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>; 3162 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>; 3163 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>; 3164 3165 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3166 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3167 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3168 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3169} 3170 3171//===----------------------------------------------------------------------===// 3172// SVE2 Widening Integer Arithmetic Group 3173//===----------------------------------------------------------------------===// 3174 3175class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3176 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3177: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3178 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3179 bits<5> Zd; 3180 bits<5> Zn; 3181 bits<5> Zm; 3182 let Inst{31-24} = 0b01000101; 3183 let Inst{23-22} = sz; 3184 let Inst{21} = 0b0; 3185 let Inst{20-16} = Zm; 3186 let Inst{15} = 0b0; 3187 let Inst{14-10} = opc; 3188 let Inst{9-5} = Zn; 3189 let Inst{4-0} = Zd; 3190} 3191 3192multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3193 SDPatternOperator op> { 3194 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3195 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3196 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3197 3198 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3199 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3200 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3201} 3202 3203multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3204 SDPatternOperator op> { 3205 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3206 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3207 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3208 3209 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3210 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3211 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3212} 3213 3214multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3215 SDPatternOperator op> { 3216 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3217 3218 // To avoid using 128 bit elements in the IR, the pattern below works with 3219 // llvm intrinsics with the _pair suffix, to reflect that 3220 // _Q is implemented as a pair of _D. 3221 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3222} 3223 3224multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3225 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3226 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3227 3228 // To avoid using 128 bit elements in the IR, the patterns below work with 3229 // llvm intrinsics with the _pair suffix, to reflect that 3230 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3231 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3232 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3233} 3234 3235//===----------------------------------------------------------------------===// 3236// SVE2 Misc Group 3237//===----------------------------------------------------------------------===// 3238 3239class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3240 ZPRRegOp zprty1, ZPRRegOp zprty2> 3241: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3242 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3243 bits<5> Zd; 3244 bits<5> Zn; 3245 bits<5> Zm; 3246 let Inst{31-24} = 0b01000101; 3247 let Inst{23-22} = sz; 3248 let Inst{21} = 0b0; 3249 let Inst{20-16} = Zm; 3250 let Inst{15-14} = 0b10; 3251 let Inst{13-10} = opc; 3252 let Inst{9-5} = Zn; 3253 let Inst{4-0} = Zd; 3254} 3255 3256multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3257 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3258 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3259 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3260 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3261 3262 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3263 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3264 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3265 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3266} 3267 3268multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3269 SDPatternOperator op> { 3270 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3271 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3272 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3273 3274 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3275 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3276 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3277} 3278 3279class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3280 ZPRRegOp zprty1, ZPRRegOp zprty2> 3281: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3282 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3283 bits<5> Zd; 3284 bits<5> Zn; 3285 bits<5> Zm; 3286 let Inst{31-24} = 0b01000101; 3287 let Inst{23-22} = sz; 3288 let Inst{21} = 0b0; 3289 let Inst{20-16} = Zm; 3290 let Inst{15-11} = 0b10010; 3291 let Inst{10} = opc; 3292 let Inst{9-5} = Zn; 3293 let Inst{4-0} = Zd; 3294 3295 let Constraints = "$Zd = $_Zd"; 3296 let DestructiveInstType = DestructiveOther; 3297 let ElementSize = ElementSizeNone; 3298} 3299 3300multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3301 SDPatternOperator op> { 3302 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3303 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3304 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3305 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3306 3307 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3308 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3309 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3310 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3311} 3312 3313class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3314 ZPRRegOp zprty1, ZPRRegOp zprty2, 3315 Operand immtype> 3316: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3317 asm, "\t$Zd, $Zn, $imm", 3318 "", []>, Sched<[]> { 3319 bits<5> Zd; 3320 bits<5> Zn; 3321 bits<5> imm; 3322 let Inst{31-23} = 0b010001010; 3323 let Inst{22} = tsz8_64{2}; 3324 let Inst{21} = 0b0; 3325 let Inst{20-19} = tsz8_64{1-0}; 3326 let Inst{18-16} = imm{2-0}; // imm3 3327 let Inst{15-12} = 0b1010; 3328 let Inst{11-10} = opc; 3329 let Inst{9-5} = Zn; 3330 let Inst{4-0} = Zd; 3331} 3332 3333multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3334 SDPatternOperator op> { 3335 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3336 ZPR16, ZPR8, vecshiftL8>; 3337 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3338 ZPR32, ZPR16, vecshiftL16> { 3339 let Inst{19} = imm{3}; 3340 } 3341 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3342 ZPR64, ZPR32, vecshiftL32> { 3343 let Inst{20-19} = imm{4-3}; 3344 } 3345 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3346 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3347 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3348} 3349 3350//===----------------------------------------------------------------------===// 3351// SVE2 Accumulate Group 3352//===----------------------------------------------------------------------===// 3353 3354class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3355 ZPRRegOp zprty, Operand immtype> 3356: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3357 asm, "\t$Zd, $Zn, $imm", 3358 "", []>, Sched<[]> { 3359 bits<5> Zd; 3360 bits<5> Zn; 3361 bits<6> imm; 3362 let Inst{31-24} = 0b01000101; 3363 let Inst{23-22} = tsz8_64{3-2}; 3364 let Inst{21} = 0b0; 3365 let Inst{20-19} = tsz8_64{1-0}; 3366 let Inst{18-16} = imm{2-0}; // imm3 3367 let Inst{15-11} = 0b11110; 3368 let Inst{10} = opc; 3369 let Inst{9-5} = Zn; 3370 let Inst{4-0} = Zd; 3371 3372 let Constraints = "$Zd = $_Zd"; 3373} 3374 3375multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3376 SDPatternOperator op> { 3377 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3378 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3379 let Inst{19} = imm{3}; 3380 } 3381 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3382 let Inst{20-19} = imm{4-3}; 3383 } 3384 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3385 let Inst{22} = imm{5}; 3386 let Inst{20-19} = imm{4-3}; 3387 } 3388 3389 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3390 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3391 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3392 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3393} 3394 3395multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3396 SDPatternOperator op> { 3397 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3398 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3399 let Inst{19} = imm{3}; 3400 } 3401 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3402 let Inst{20-19} = imm{4-3}; 3403 } 3404 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3405 let Inst{22} = imm{5}; 3406 let Inst{20-19} = imm{4-3}; 3407 } 3408 3409 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3410 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3411 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3412 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3413} 3414 3415class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3416 ZPRRegOp zprty, Operand immtype> 3417: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3418 asm, "\t$Zda, $Zn, $imm", 3419 "", []>, Sched<[]> { 3420 bits<5> Zda; 3421 bits<5> Zn; 3422 bits<6> imm; 3423 let Inst{31-24} = 0b01000101; 3424 let Inst{23-22} = tsz8_64{3-2}; 3425 let Inst{21} = 0b0; 3426 let Inst{20-19} = tsz8_64{1-0}; 3427 let Inst{18-16} = imm{2-0}; // imm3 3428 let Inst{15-12} = 0b1110; 3429 let Inst{11-10} = opc; 3430 let Inst{9-5} = Zn; 3431 let Inst{4-0} = Zda; 3432 3433 let Constraints = "$Zda = $_Zda"; 3434 let DestructiveInstType = DestructiveOther; 3435 let ElementSize = ElementSizeNone; 3436} 3437 3438multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3439 SDPatternOperator op> { 3440 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3441 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3442 let Inst{19} = imm{3}; 3443 } 3444 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3445 let Inst{20-19} = imm{4-3}; 3446 } 3447 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3448 let Inst{22} = imm{5}; 3449 let Inst{20-19} = imm{4-3}; 3450 } 3451 3452 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3453 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3454 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3455 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3456} 3457 3458class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3459: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3460 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3461 bits<5> Zdn; 3462 bits<5> Zm; 3463 bit rot; 3464 let Inst{31-24} = 0b01000101; 3465 let Inst{23-22} = sz; 3466 let Inst{21-17} = 0b00000; 3467 let Inst{16} = opc; 3468 let Inst{15-11} = 0b11011; 3469 let Inst{10} = rot; 3470 let Inst{9-5} = Zm; 3471 let Inst{4-0} = Zdn; 3472 3473 let Constraints = "$Zdn = $_Zdn"; 3474 let DestructiveInstType = DestructiveOther; 3475 let ElementSize = ElementSizeNone; 3476} 3477 3478multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3479 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3480 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3481 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3482 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3483 3484 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3485 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3486 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3487 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3488} 3489 3490class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3491 ZPRRegOp zprty1, ZPRRegOp zprty2> 3492: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3493 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3494 bits<5> Zda; 3495 bits<5> Zn; 3496 bits<5> Zm; 3497 let Inst{31-24} = 0b01000101; 3498 let Inst{23-22} = sz; 3499 let Inst{21} = 0b0; 3500 let Inst{20-16} = Zm; 3501 let Inst{15-14} = 0b11; 3502 let Inst{13-10} = opc; 3503 let Inst{9-5} = Zn; 3504 let Inst{4-0} = Zda; 3505 3506 let Constraints = "$Zda = $_Zda"; 3507 let DestructiveInstType = DestructiveOther; 3508 let ElementSize = ElementSizeNone; 3509} 3510 3511multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3512 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3513 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3514 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3515 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3516 3517 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3518 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3519 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3520 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3521} 3522 3523multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3524 SDPatternOperator op> { 3525 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3526 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3527 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3528 3529 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3530 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3531 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3532} 3533 3534multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3535 SDPatternOperator op> { 3536 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3537 ZPR32, ZPR32>; 3538 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3539 ZPR64, ZPR64>; 3540 3541 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3542 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3543} 3544 3545//===----------------------------------------------------------------------===// 3546// SVE2 Narrowing Group 3547//===----------------------------------------------------------------------===// 3548 3549class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3550 string asm, ZPRRegOp zprty1, 3551 ZPRRegOp zprty2, Operand immtype> 3552: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3553 asm, "\t$Zd, $Zn, $imm", 3554 "", []>, Sched<[]> { 3555 bits<5> Zd; 3556 bits<5> Zn; 3557 bits<5> imm; 3558 let Inst{31-23} = 0b010001010; 3559 let Inst{22} = tsz8_64{2}; 3560 let Inst{21} = 0b1; 3561 let Inst{20-19} = tsz8_64{1-0}; 3562 let Inst{18-16} = imm{2-0}; // imm3 3563 let Inst{15-14} = 0b00; 3564 let Inst{13-11} = opc; 3565 let Inst{10} = 0b0; 3566 let Inst{9-5} = Zn; 3567 let Inst{4-0} = Zd; 3568} 3569 3570multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3571 SDPatternOperator op> { 3572 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3573 tvecshiftR8>; 3574 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3575 tvecshiftR16> { 3576 let Inst{19} = imm{3}; 3577 } 3578 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3579 tvecshiftR32> { 3580 let Inst{20-19} = imm{4-3}; 3581 } 3582 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3583 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3584 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3585} 3586 3587class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3588 string asm, ZPRRegOp zprty1, 3589 ZPRRegOp zprty2, Operand immtype> 3590: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3591 asm, "\t$Zd, $Zn, $imm", 3592 "", []>, Sched<[]> { 3593 bits<5> Zd; 3594 bits<5> Zn; 3595 bits<5> imm; 3596 let Inst{31-23} = 0b010001010; 3597 let Inst{22} = tsz8_64{2}; 3598 let Inst{21} = 0b1; 3599 let Inst{20-19} = tsz8_64{1-0}; 3600 let Inst{18-16} = imm{2-0}; // imm3 3601 let Inst{15-14} = 0b00; 3602 let Inst{13-11} = opc; 3603 let Inst{10} = 0b1; 3604 let Inst{9-5} = Zn; 3605 let Inst{4-0} = Zd; 3606 3607 let Constraints = "$Zd = $_Zd"; 3608} 3609 3610multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 3611 SDPatternOperator op> { 3612 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 3613 tvecshiftR8>; 3614 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 3615 tvecshiftR16> { 3616 let Inst{19} = imm{3}; 3617 } 3618 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 3619 tvecshiftR32> { 3620 let Inst{20-19} = imm{4-3}; 3621 } 3622 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3623 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3624 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3625} 3626 3627class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 3628 ZPRRegOp zprty1, ZPRRegOp zprty2> 3629: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3630 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3631 bits<5> Zd; 3632 bits<5> Zn; 3633 bits<5> Zm; 3634 let Inst{31-24} = 0b01000101; 3635 let Inst{23-22} = sz; 3636 let Inst{21} = 0b1; 3637 let Inst{20-16} = Zm; 3638 let Inst{15-13} = 0b011; 3639 let Inst{12-11} = opc; // S, R 3640 let Inst{10} = 0b0; // Top 3641 let Inst{9-5} = Zn; 3642 let Inst{4-0} = Zd; 3643} 3644 3645multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 3646 SDPatternOperator op> { 3647 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 3648 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 3649 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 3650 3651 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3652 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3653 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3654} 3655 3656class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 3657 ZPRRegOp zprty1, ZPRRegOp zprty2> 3658: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3659 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3660 bits<5> Zd; 3661 bits<5> Zn; 3662 bits<5> Zm; 3663 let Inst{31-24} = 0b01000101; 3664 let Inst{23-22} = sz; 3665 let Inst{21} = 0b1; 3666 let Inst{20-16} = Zm; 3667 let Inst{15-13} = 0b011; 3668 let Inst{12-11} = opc; // S, R 3669 let Inst{10} = 0b1; // Top 3670 let Inst{9-5} = Zn; 3671 let Inst{4-0} = Zd; 3672 3673 let Constraints = "$Zd = $_Zd"; 3674} 3675 3676multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 3677 SDPatternOperator op> { 3678 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 3679 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 3680 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 3681 3682 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3683 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3684 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3685} 3686 3687class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 3688 ZPRRegOp zprty1, ZPRRegOp zprty2> 3689: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 3690 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3691 bits<5> Zd; 3692 bits<5> Zn; 3693 let Inst{31-23} = 0b010001010; 3694 let Inst{22} = tsz8_64{2}; 3695 let Inst{21} = 0b1; 3696 let Inst{20-19} = tsz8_64{1-0}; 3697 let Inst{18-13} = 0b000010; 3698 let Inst{12-11} = opc; 3699 let Inst{10} = 0b0; 3700 let Inst{9-5} = Zn; 3701 let Inst{4-0} = Zd; 3702} 3703 3704multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 3705 SDPatternOperator op> { 3706 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 3707 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 3708 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 3709 3710 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 3711 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 3712 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 3713} 3714 3715class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 3716 ZPRRegOp zprty1, ZPRRegOp zprty2> 3717: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 3718 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3719 bits<5> Zd; 3720 bits<5> Zn; 3721 let Inst{31-23} = 0b010001010; 3722 let Inst{22} = tsz8_64{2}; 3723 let Inst{21} = 0b1; 3724 let Inst{20-19} = tsz8_64{1-0}; 3725 let Inst{18-13} = 0b000010; 3726 let Inst{12-11} = opc; 3727 let Inst{10} = 0b1; 3728 let Inst{9-5} = Zn; 3729 let Inst{4-0} = Zd; 3730 3731 let Constraints = "$Zd = $_Zd"; 3732} 3733 3734multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 3735 SDPatternOperator op> { 3736 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 3737 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 3738 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 3739 3740 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 3741 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 3742 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 3743} 3744 3745//===----------------------------------------------------------------------===// 3746// SVE Integer Arithmetic - Unary Predicated Group 3747//===----------------------------------------------------------------------===// 3748 3749class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 3750 string asm, ZPRRegOp zprty> 3751: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3752 asm, "\t$Zd, $Pg/m, $Zn", 3753 "", 3754 []>, Sched<[]> { 3755 bits<3> Pg; 3756 bits<5> Zd; 3757 bits<5> Zn; 3758 let Inst{31-24} = 0b00000100; 3759 let Inst{23-22} = sz8_64; 3760 let Inst{21-20} = 0b01; 3761 let Inst{19} = opc{0}; 3762 let Inst{18-16} = opc{3-1}; 3763 let Inst{15-13} = 0b101; 3764 let Inst{12-10} = Pg; 3765 let Inst{9-5} = Zn; 3766 let Inst{4-0} = Zd; 3767 3768 let Constraints = "$Zd = $_Zd"; 3769 let DestructiveInstType = DestructiveOther; 3770 let ElementSize = zprty.ElementSize; 3771} 3772 3773multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 3774 SDPatternOperator op> { 3775 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>; 3776 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>; 3777 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>; 3778 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3779 3780 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3781 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3782 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3783 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3784} 3785 3786multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 3787 SDPatternOperator op> { 3788 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>; 3789 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>; 3790 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3791 3792 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 3793 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 3794 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 3795} 3796 3797multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 3798 SDPatternOperator op> { 3799 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>; 3800 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3801 3802 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 3803 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 3804} 3805 3806multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 3807 SDPatternOperator op> { 3808 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>; 3809 3810 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 3811} 3812 3813multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 3814 SDPatternOperator op> { 3815 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>; 3816 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>; 3817 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>; 3818 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>; 3819 3820 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3821 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3822 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3823 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3824} 3825 3826multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 3827 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>; 3828 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>; 3829 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>; 3830 3831 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 3832 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 3833 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 3834 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 3835 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 3836 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 3837} 3838 3839//===----------------------------------------------------------------------===// 3840// SVE Integer Wide Immediate - Unpredicated Group 3841//===----------------------------------------------------------------------===// 3842class sve_int_dup_imm<bits<2> sz8_64, string asm, 3843 ZPRRegOp zprty, Operand immtype> 3844: I<(outs zprty:$Zd), (ins immtype:$imm), 3845 asm, "\t$Zd, $imm", 3846 "", 3847 []>, Sched<[]> { 3848 bits<5> Zd; 3849 bits<9> imm; 3850 let Inst{31-24} = 0b00100101; 3851 let Inst{23-22} = sz8_64; 3852 let Inst{21-14} = 0b11100011; 3853 let Inst{13} = imm{8}; // sh 3854 let Inst{12-5} = imm{7-0}; // imm8 3855 let Inst{4-0} = Zd; 3856 3857 let isReMaterializable = 1; 3858} 3859 3860multiclass sve_int_dup_imm<string asm> { 3861 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 3862 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 3863 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 3864 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 3865 3866 def : InstAlias<"mov $Zd, $imm", 3867 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 3868 def : InstAlias<"mov $Zd, $imm", 3869 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 3870 def : InstAlias<"mov $Zd, $imm", 3871 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 3872 def : InstAlias<"mov $Zd, $imm", 3873 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 3874 3875 def : InstAlias<"fmov $Zd, #0.0", 3876 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 3877 def : InstAlias<"fmov $Zd, #0.0", 3878 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 3879 def : InstAlias<"fmov $Zd, #0.0", 3880 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 3881} 3882 3883class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 3884 string asm, ZPRRegOp zprty> 3885: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 3886 asm, "\t$Zd, $imm8", 3887 "", 3888 []>, Sched<[]> { 3889 bits<5> Zd; 3890 bits<8> imm8; 3891 let Inst{31-24} = 0b00100101; 3892 let Inst{23-22} = sz8_64; 3893 let Inst{21-14} = 0b11100111; 3894 let Inst{13} = 0b0; 3895 let Inst{12-5} = imm8; 3896 let Inst{4-0} = Zd; 3897 3898 let isReMaterializable = 1; 3899} 3900 3901multiclass sve_int_dup_fpimm<string asm> { 3902 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 3903 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 3904 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 3905 3906 def : InstAlias<"fmov $Zd, $imm8", 3907 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 3908 def : InstAlias<"fmov $Zd, $imm8", 3909 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 3910 def : InstAlias<"fmov $Zd, $imm8", 3911 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 3912} 3913 3914class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 3915 ZPRRegOp zprty, Operand immtype> 3916: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 3917 asm, "\t$Zdn, $_Zdn, $imm", 3918 "", 3919 []>, Sched<[]> { 3920 bits<5> Zdn; 3921 bits<9> imm; 3922 let Inst{31-24} = 0b00100101; 3923 let Inst{23-22} = sz8_64; 3924 let Inst{21-19} = 0b100; 3925 let Inst{18-16} = opc; 3926 let Inst{15-14} = 0b11; 3927 let Inst{13} = imm{8}; // sh 3928 let Inst{12-5} = imm{7-0}; // imm8 3929 let Inst{4-0} = Zdn; 3930 3931 let Constraints = "$Zdn = $_Zdn"; 3932 let DestructiveInstType = DestructiveOther; 3933 let ElementSize = ElementSizeNone; 3934} 3935 3936multiclass sve_int_arith_imm0<bits<3> opc, string asm, 3937 SDPatternOperator op, SDPatternOperator int_op> { 3938 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 3939 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 3940 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 3941 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 3942 3943 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 3944 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 3945 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 3946 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 3947 3948 // Intrinsic version 3949 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, int_op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 3950 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, int_op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 3951 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, int_op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 3952 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, int_op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 3953} 3954 3955multiclass sve_int_arith_imm0_subr<bits<3> opc, string asm, SDPatternOperator op> { 3956 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 3957 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 3958 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 3959 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 3960 3961 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 3962 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 3963 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 3964 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 3965} 3966 3967class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 3968 ZPRRegOp zprty, Operand immtype> 3969: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 3970 asm, "\t$Zdn, $_Zdn, $imm", 3971 "", 3972 []>, Sched<[]> { 3973 bits<5> Zdn; 3974 bits<8> imm; 3975 let Inst{31-24} = 0b00100101; 3976 let Inst{23-22} = sz8_64; 3977 let Inst{21-16} = opc; 3978 let Inst{15-13} = 0b110; 3979 let Inst{12-5} = imm; 3980 let Inst{4-0} = Zdn; 3981 3982 let Constraints = "$Zdn = $_Zdn"; 3983 let DestructiveInstType = DestructiveOther; 3984 let ElementSize = ElementSizeNone; 3985} 3986 3987multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 3988 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 3989 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 3990 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 3991 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 3992 3993 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 3994 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 3995 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 3996 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 3997} 3998 3999multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4000 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4001 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4002 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4003 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4004 4005 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4006 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4007 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4008 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4009} 4010 4011multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4012 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 4013 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 4014 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 4015 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 4016 4017 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 4018 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 4019 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 4020 def : SVE_1_Op_Imm_Arith_Pred_Pat<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 4021} 4022 4023//===----------------------------------------------------------------------===// 4024// SVE Bitwise Logical - Unpredicated Group 4025//===----------------------------------------------------------------------===// 4026 4027class sve_int_bin_cons_log<bits<2> opc, string asm> 4028: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4029 asm, "\t$Zd, $Zn, $Zm", 4030 "", 4031 []>, Sched<[]> { 4032 bits<5> Zd; 4033 bits<5> Zm; 4034 bits<5> Zn; 4035 let Inst{31-24} = 0b00000100; 4036 let Inst{23-22} = opc{1-0}; 4037 let Inst{21} = 0b1; 4038 let Inst{20-16} = Zm; 4039 let Inst{15-10} = 0b001100; 4040 let Inst{9-5} = Zn; 4041 let Inst{4-0} = Zd; 4042} 4043 4044multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4045 def NAME : sve_int_bin_cons_log<opc, asm>; 4046 4047 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4048 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4049 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4050 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4051 4052 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4053 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4054 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4055 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4056 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4057 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4058} 4059 4060class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4061: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4062 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4063 "", 4064 []>, Sched<[]> { 4065 bits<5> Zdn; 4066 bits<5> Zk; 4067 bits<5> Zm; 4068 let Inst{31-24} = 0b00000100; 4069 let Inst{23-22} = opc{2-1}; 4070 let Inst{21} = 0b1; 4071 let Inst{20-16} = Zm; 4072 let Inst{15-11} = 0b00111; 4073 let Inst{10} = opc{0}; 4074 let Inst{9-5} = Zk; 4075 let Inst{4-0} = Zdn; 4076 4077 let Constraints = "$Zdn = $_Zdn"; 4078 let DestructiveInstType = DestructiveOther; 4079 let ElementSize = ElementSizeNone; 4080} 4081 4082multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op> { 4083 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4084 4085 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4086 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4087 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4088 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4089 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4090 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4091 4092 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4093 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4094 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4095 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4096} 4097 4098class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4099 ZPRRegOp zprty, Operand immtype> 4100: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4101 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4102 "", 4103 []>, Sched<[]> { 4104 bits<5> Zdn; 4105 bits<5> Zm; 4106 bits<6> imm; 4107 let Inst{31-24} = 0b00000100; 4108 let Inst{23-22} = tsz8_64{3-2}; 4109 let Inst{21} = 0b1; 4110 let Inst{20-19} = tsz8_64{1-0}; 4111 let Inst{18-16} = imm{2-0}; // imm3 4112 let Inst{15-10} = 0b001101; 4113 let Inst{9-5} = Zm; 4114 let Inst{4-0} = Zdn; 4115 4116 let Constraints = "$Zdn = $_Zdn"; 4117 let DestructiveInstType = DestructiveOther; 4118 let ElementSize = ElementSizeNone; 4119} 4120 4121multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4122 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4123 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4124 let Inst{19} = imm{3}; 4125 } 4126 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4127 let Inst{20-19} = imm{4-3}; 4128 } 4129 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4130 let Inst{22} = imm{5}; 4131 let Inst{20-19} = imm{4-3}; 4132 } 4133 4134 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4135 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4136 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4137 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4138} 4139 4140//===----------------------------------------------------------------------===// 4141// SVE Integer Wide Immediate - Predicated Group 4142//===----------------------------------------------------------------------===// 4143 4144class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4145 string asm, ZPRRegOp zprty> 4146: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4147 asm, "\t$Zd, $Pg/m, $imm8", 4148 "", 4149 []>, Sched<[]> { 4150 bits<4> Pg; 4151 bits<5> Zd; 4152 bits<8> imm8; 4153 let Inst{31-24} = 0b00000101; 4154 let Inst{23-22} = sz; 4155 let Inst{21-20} = 0b01; 4156 let Inst{19-16} = Pg; 4157 let Inst{15-13} = 0b110; 4158 let Inst{12-5} = imm8; 4159 let Inst{4-0} = Zd; 4160 4161 let Constraints = "$Zd = $_Zd"; 4162 let DestructiveInstType = DestructiveOther; 4163 let ElementSize = zprty.ElementSize; 4164} 4165 4166multiclass sve_int_dup_fpimm_pred<string asm> { 4167 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4168 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4169 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4170 4171 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4172 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4173 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4174 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4175 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4176 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4177} 4178 4179class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4180 ZPRRegOp zprty, string pred_qual, dag iops> 4181: I<(outs zprty:$Zd), iops, 4182 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4183 "", []>, Sched<[]> { 4184 bits<5> Zd; 4185 bits<4> Pg; 4186 bits<9> imm; 4187 let Inst{31-24} = 0b00000101; 4188 let Inst{23-22} = sz8_64; 4189 let Inst{21-20} = 0b01; 4190 let Inst{19-16} = Pg; 4191 let Inst{15} = 0b0; 4192 let Inst{14} = m; 4193 let Inst{13} = imm{8}; // sh 4194 let Inst{12-5} = imm{7-0}; // imm8 4195 let Inst{4-0} = Zd; 4196 4197 let DestructiveInstType = DestructiveOther; 4198 let ElementSize = zprty.ElementSize; 4199} 4200 4201multiclass sve_int_dup_imm_pred_merge_inst< 4202 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4203 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4204 let Constraints = "$Zd = $_Zd" in 4205 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4206 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4207 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4208 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4209 def : Pat<(intty 4210 (vselect predty:$Pg, 4211 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4212 intty:$Zd)), 4213 (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; 4214} 4215 4216multiclass sve_int_dup_imm_pred_merge<string asm> { 4217 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4218 i32, cpy_imm8_opt_lsl_i8>; 4219 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4220 i32, cpy_imm8_opt_lsl_i16>; 4221 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4222 i32, cpy_imm8_opt_lsl_i32>; 4223 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4224 i64, cpy_imm8_opt_lsl_i64>; 4225 4226 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4227 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4228 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4229 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4230 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4231 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4232} 4233 4234multiclass sve_int_dup_imm_pred_zero_inst< 4235 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4236 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4237 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4238 (ins PPRAny:$Pg, cpyimm:$imm)>; 4239 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4240 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4241 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4242 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4243 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4244 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4245 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4246 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4247 def : Pat<(intty 4248 (vselect predty:$Pg, 4249 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4250 (intty (AArch64dup (scalarty 0))))), 4251 (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>; 4252} 4253 4254multiclass sve_int_dup_imm_pred_zero<string asm> { 4255 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4256 i32, cpy_imm8_opt_lsl_i8>; 4257 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4258 i32, cpy_imm8_opt_lsl_i16>; 4259 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4260 i32, cpy_imm8_opt_lsl_i32>; 4261 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4262 i64, cpy_imm8_opt_lsl_i64>; 4263} 4264 4265//===----------------------------------------------------------------------===// 4266// SVE Integer Compare - Vectors Group 4267//===----------------------------------------------------------------------===// 4268 4269class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4270 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4271: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4272 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4273 "", 4274 []>, Sched<[]> { 4275 bits<4> Pd; 4276 bits<3> Pg; 4277 bits<5> Zm; 4278 bits<5> Zn; 4279 let Inst{31-24} = 0b00100100; 4280 let Inst{23-22} = sz8_64; 4281 let Inst{21} = 0b0; 4282 let Inst{20-16} = Zm; 4283 let Inst{15} = opc{2}; 4284 let Inst{14} = cmp_1; 4285 let Inst{13} = opc{1}; 4286 let Inst{12-10} = Pg; 4287 let Inst{9-5} = Zn; 4288 let Inst{4} = opc{0}; 4289 let Inst{3-0} = Pd; 4290 4291 let Defs = [NZCV]; 4292 let ElementSize = pprty.ElementSize; 4293 let isPTestLike = 1; 4294} 4295 4296multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4297 ValueType intvt, sve_int_cmp cmp> { 4298 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4299 (cmp $Op1, $Op2, $Op3)>; 4300 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4301 (cmp $Op1, $Op3, $Op2)>; 4302} 4303 4304multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4305 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4306 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4307 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4308 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4309 4310 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4311 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4312 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4313 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4314} 4315 4316multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4317 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4318 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4319 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4320 4321 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4322 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4323 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4324} 4325 4326multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4327 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4328 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4329 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4330 4331 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4332 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4333 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4334} 4335 4336 4337//===----------------------------------------------------------------------===// 4338// SVE Integer Compare - Signed Immediate Group 4339//===----------------------------------------------------------------------===// 4340 4341class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4342 ZPRRegOp zprty, 4343 Operand immtype> 4344: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4345 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4346 "", 4347 []>, Sched<[]> { 4348 bits<4> Pd; 4349 bits<3> Pg; 4350 bits<5> Zn; 4351 bits<5> imm5; 4352 let Inst{31-24} = 0b00100101; 4353 let Inst{23-22} = sz8_64; 4354 let Inst{21} = 0b0; 4355 let Inst{20-16} = imm5; 4356 let Inst{15} = opc{2}; 4357 let Inst{14} = 0b0; 4358 let Inst{13} = opc{1}; 4359 let Inst{12-10} = Pg; 4360 let Inst{9-5} = Zn; 4361 let Inst{4} = opc{0}; 4362 let Inst{3-0} = Pd; 4363 4364 let Defs = [NZCV]; 4365 let ElementSize = pprty.ElementSize; 4366 let isPTestLike = 1; 4367} 4368 4369multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4370 ValueType predvt, ValueType intvt, 4371 Operand immtype, Instruction cmp> { 4372 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4373 (intvt ZPR:$Zs1), 4374 (intvt (AArch64dup (immtype:$imm))), 4375 cc)), 4376 (cmp $Pg, $Zs1, immtype:$imm)>; 4377 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4378 (intvt (AArch64dup (immtype:$imm))), 4379 (intvt ZPR:$Zs1), 4380 commuted_cc)), 4381 (cmp $Pg, $Zs1, immtype:$imm)>; 4382} 4383 4384multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4385 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4386 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4387 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4388 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4389 4390 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4391 !cast<Instruction>(NAME # _B)>; 4392 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4393 !cast<Instruction>(NAME # _H)>; 4394 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4395 !cast<Instruction>(NAME # _S)>; 4396 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4397 !cast<Instruction>(NAME # _D)>; 4398} 4399 4400 4401//===----------------------------------------------------------------------===// 4402// SVE Integer Compare - Unsigned Immediate Group 4403//===----------------------------------------------------------------------===// 4404 4405class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4406 ZPRRegOp zprty, Operand immtype> 4407: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4408 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4409 "", 4410 []>, Sched<[]> { 4411 bits<4> Pd; 4412 bits<3> Pg; 4413 bits<5> Zn; 4414 bits<7> imm7; 4415 let Inst{31-24} = 0b00100100; 4416 let Inst{23-22} = sz8_64; 4417 let Inst{21} = 1; 4418 let Inst{20-14} = imm7; 4419 let Inst{13} = opc{1}; 4420 let Inst{12-10} = Pg; 4421 let Inst{9-5} = Zn; 4422 let Inst{4} = opc{0}; 4423 let Inst{3-0} = Pd; 4424 4425 let Defs = [NZCV]; 4426 let ElementSize = pprty.ElementSize; 4427 let isPTestLike = 1; 4428} 4429 4430multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4431 CondCode commuted_cc> { 4432 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4433 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4434 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4435 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4436 4437 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4438 !cast<Instruction>(NAME # _B)>; 4439 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4440 !cast<Instruction>(NAME # _H)>; 4441 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4442 !cast<Instruction>(NAME # _S)>; 4443 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4444 !cast<Instruction>(NAME # _D)>; 4445} 4446 4447 4448//===----------------------------------------------------------------------===// 4449// SVE Integer Compare - Scalars Group 4450//===----------------------------------------------------------------------===// 4451 4452class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4453: I<(outs), (ins rt:$Rn, rt:$Rm), 4454 asm, "\t$Rn, $Rm", 4455 "", 4456 []>, Sched<[]> { 4457 bits<5> Rm; 4458 bits<5> Rn; 4459 let Inst{31-23} = 0b001001011; 4460 let Inst{22} = sz; 4461 let Inst{21} = 0b1; 4462 let Inst{20-16} = Rm; 4463 let Inst{15-10} = 0b001000; 4464 let Inst{9-5} = Rn; 4465 let Inst{4} = opc; 4466 let Inst{3-0} = 0b0000; 4467 4468 let Defs = [NZCV]; 4469} 4470 4471class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4472 RegisterClass gprty, PPRRegOp pprty> 4473: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4474 asm, "\t$Pd, $Rn, $Rm", 4475 "", []>, Sched<[]> { 4476 bits<4> Pd; 4477 bits<5> Rm; 4478 bits<5> Rn; 4479 let Inst{31-24} = 0b00100101; 4480 let Inst{23-22} = sz8_64; 4481 let Inst{21} = 0b1; 4482 let Inst{20-16} = Rm; 4483 let Inst{15-13} = 0b000; 4484 let Inst{12-10} = opc{3-1}; 4485 let Inst{9-5} = Rn; 4486 let Inst{4} = opc{0}; 4487 let Inst{3-0} = Pd; 4488 4489 let Defs = [NZCV]; 4490 let ElementSize = pprty.ElementSize; 4491 let isWhile = 1; 4492} 4493 4494multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4495 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 4496 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 4497 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 4498 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 4499 4500 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4501 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4502 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4503 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 4504} 4505 4506multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 4507 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 4508 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 4509 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 4510 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 4511 4512 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 4513 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 4514 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 4515 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 4516} 4517 4518class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 4519 PPRRegOp pprty> 4520: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 4521 asm, "\t$Pd, $Rn, $Rm", 4522 "", []>, Sched<[]> { 4523 bits<4> Pd; 4524 bits<5> Rm; 4525 bits<5> Rn; 4526 let Inst{31-24} = 0b00100101; 4527 let Inst{23-22} = sz8_64; 4528 let Inst{21} = 0b1; 4529 let Inst{20-16} = Rm; 4530 let Inst{15-10} = 0b001100; 4531 let Inst{9-5} = Rn; 4532 let Inst{4} = rw; 4533 let Inst{3-0} = Pd; 4534 4535 let Defs = [NZCV]; 4536 let ElementSize = pprty.ElementSize; 4537 let isWhile = 1; 4538} 4539 4540multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 4541 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 4542 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 4543 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 4544 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 4545 4546 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 4547 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 4548 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 4549 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 4550} 4551 4552//===----------------------------------------------------------------------===// 4553// SVE Floating Point Fast Reduction Group 4554//===----------------------------------------------------------------------===// 4555 4556class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 4557 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4558: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 4559 asm, "\t$Vd, $Pg, $Zn", 4560 "", 4561 []>, Sched<[]> { 4562 bits<5> Zn; 4563 bits<5> Vd; 4564 bits<3> Pg; 4565 let Inst{31-24} = 0b01100101; 4566 let Inst{23-22} = sz; 4567 let Inst{21-19} = 0b000; 4568 let Inst{18-16} = opc; 4569 let Inst{15-13} = 0b001; 4570 let Inst{12-10} = Pg; 4571 let Inst{9-5} = Zn; 4572 let Inst{4-0} = Vd; 4573} 4574 4575multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 4576 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 4577 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 4578 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 4579 4580 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4581 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4582 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4583 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4584 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4585 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4586} 4587 4588//===----------------------------------------------------------------------===// 4589// SVE Floating Point Accumulating Reduction Group 4590//===----------------------------------------------------------------------===// 4591 4592class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 4593 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4594: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 4595 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 4596 "", 4597 []>, 4598 Sched<[]> { 4599 bits<3> Pg; 4600 bits<5> Vdn; 4601 bits<5> Zm; 4602 let Inst{31-24} = 0b01100101; 4603 let Inst{23-22} = sz; 4604 let Inst{21-19} = 0b011; 4605 let Inst{18-16} = opc; 4606 let Inst{15-13} = 0b001; 4607 let Inst{12-10} = Pg; 4608 let Inst{9-5} = Zm; 4609 let Inst{4-0} = Vdn; 4610 4611 let Constraints = "$Vdn = $_Vdn"; 4612} 4613 4614multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 4615 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 4616 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 4617 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 4618 4619 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 4620 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 4621 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4622 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 4623 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4624 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4625} 4626 4627//===----------------------------------------------------------------------===// 4628// SVE Floating Point Compare - Vectors Group 4629//===----------------------------------------------------------------------===// 4630 4631class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4632 ZPRRegOp zprty> 4633: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 4634 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4635 "", 4636 []>, Sched<[]> { 4637 bits<4> Pd; 4638 bits<3> Pg; 4639 bits<5> Zm; 4640 bits<5> Zn; 4641 let Inst{31-24} = 0b01100101; 4642 let Inst{23-22} = sz; 4643 let Inst{21} = 0b0; 4644 let Inst{20-16} = Zm; 4645 let Inst{15} = opc{2}; 4646 let Inst{14} = 0b1; 4647 let Inst{13} = opc{1}; 4648 let Inst{12-10} = Pg; 4649 let Inst{9-5} = Zn; 4650 let Inst{4} = opc{0}; 4651 let Inst{3-0} = Pd; 4652} 4653 4654multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 4655 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4656 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4657 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4658 4659 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4660 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4661 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4662} 4663 4664multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, SDPatternOperator op, 4665 SDPatternOperator op_nopred> 4666: sve_fp_3op_p_pd<opc, asm, op> { 4667 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8f16, nxv8f16, 4668 !cast<Instruction>(NAME # _H), PTRUE_H>; 4669 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f16, nxv4f16, 4670 !cast<Instruction>(NAME # _H), PTRUE_S>; 4671 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f16, nxv2f16, 4672 !cast<Instruction>(NAME # _H), PTRUE_D>; 4673 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4f32, nxv4f32, 4674 !cast<Instruction>(NAME # _S), PTRUE_S>; 4675 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f32, nxv2f32, 4676 !cast<Instruction>(NAME # _S), PTRUE_D>; 4677 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2f64, nxv2f64, 4678 !cast<Instruction>(NAME # _D), PTRUE_D>; 4679} 4680 4681//===----------------------------------------------------------------------===// 4682// SVE Floating Point Compare - with Zero Group 4683//===----------------------------------------------------------------------===// 4684 4685class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4686 ZPRRegOp zprty> 4687: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 4688 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 4689 "", 4690 []>, Sched<[]> { 4691 bits<4> Pd; 4692 bits<3> Pg; 4693 bits<5> Zn; 4694 let Inst{31-24} = 0b01100101; 4695 let Inst{23-22} = sz; 4696 let Inst{21-18} = 0b0100; 4697 let Inst{17-16} = opc{2-1}; 4698 let Inst{15-13} = 0b001; 4699 let Inst{12-10} = Pg; 4700 let Inst{9-5} = Zn; 4701 let Inst{4} = opc{0}; 4702 let Inst{3-0} = Pd; 4703} 4704 4705multiclass sve_fp_2op_p_pd<bits<3> opc, string asm> { 4706 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4707 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4708 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4709} 4710 4711 4712//===----------------------------------------------------------------------===// 4713//SVE Index Generation Group 4714//===----------------------------------------------------------------------===// 4715 4716class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4717 Operand imm_ty> 4718: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 4719 asm, "\t$Zd, $imm5, $imm5b", 4720 "", []>, Sched<[]> { 4721 bits<5> Zd; 4722 bits<5> imm5; 4723 bits<5> imm5b; 4724 let Inst{31-24} = 0b00000100; 4725 let Inst{23-22} = sz8_64; 4726 let Inst{21} = 0b1; 4727 let Inst{20-16} = imm5b; 4728 let Inst{15-10} = 0b010000; 4729 let Inst{9-5} = imm5; 4730 let Inst{4-0} = Zd; 4731} 4732 4733multiclass sve_int_index_ii<string asm, SDPatternOperator op> { 4734 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 4735 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 4736 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 4737 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 4738 4739 def : Pat<(nxv16i8 (op simm5_8b:$imm5, simm5_8b:$imm5b)), 4740 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, simm5_8b:$imm5b)>; 4741 def : Pat<(nxv8i16 (op simm5_16b:$imm5, simm5_16b:$imm5b)), 4742 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, simm5_16b:$imm5b)>; 4743 def : Pat<(nxv4i32 (op simm5_32b:$imm5, simm5_32b:$imm5b)), 4744 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 4745 def : Pat<(nxv2i64 (op simm5_64b:$imm5, simm5_64b:$imm5b)), 4746 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 4747} 4748 4749class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4750 RegisterClass srcRegType, Operand imm_ty> 4751: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 4752 asm, "\t$Zd, $imm5, $Rm", 4753 "", []>, Sched<[]> { 4754 bits<5> Rm; 4755 bits<5> Zd; 4756 bits<5> imm5; 4757 let Inst{31-24} = 0b00000100; 4758 let Inst{23-22} = sz8_64; 4759 let Inst{21} = 0b1; 4760 let Inst{20-16} = Rm; 4761 let Inst{15-10} = 0b010010; 4762 let Inst{9-5} = imm5; 4763 let Inst{4-0} = Zd; 4764} 4765 4766multiclass sve_int_index_ir<string asm, SDPatternOperator op> { 4767 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 4768 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 4769 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 4770 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 4771 4772 def : Pat<(nxv16i8 (op simm5_8b:$imm5, GPR32:$Rm)), 4773 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 4774 def : Pat<(nxv8i16 (op simm5_16b:$imm5, GPR32:$Rm)), 4775 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 4776 def : Pat<(nxv4i32 (op simm5_32b:$imm5, GPR32:$Rm)), 4777 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 4778 def : Pat<(nxv2i64 (op simm5_64b:$imm5, GPR64:$Rm)), 4779 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 4780} 4781 4782class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4783 RegisterClass srcRegType, Operand imm_ty> 4784: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 4785 asm, "\t$Zd, $Rn, $imm5", 4786 "", []>, Sched<[]> { 4787 bits<5> Rn; 4788 bits<5> Zd; 4789 bits<5> imm5; 4790 let Inst{31-24} = 0b00000100; 4791 let Inst{23-22} = sz8_64; 4792 let Inst{21} = 0b1; 4793 let Inst{20-16} = imm5; 4794 let Inst{15-10} = 0b010001; 4795 let Inst{9-5} = Rn; 4796 let Inst{4-0} = Zd; 4797} 4798 4799multiclass sve_int_index_ri<string asm, SDPatternOperator op> { 4800 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 4801 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 4802 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 4803 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 4804 4805 def : Pat<(nxv16i8 (op GPR32:$Rm, simm5_8b:$imm5)), 4806 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, simm5_8b:$imm5)>; 4807 def : Pat<(nxv8i16 (op GPR32:$Rm, simm5_16b:$imm5)), 4808 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, simm5_16b:$imm5)>; 4809 def : Pat<(nxv4i32 (op GPR32:$Rm, simm5_32b:$imm5)), 4810 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 4811 def : Pat<(nxv2i64 (op GPR64:$Rm, simm5_64b:$imm5)), 4812 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 4813} 4814 4815class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4816 RegisterClass srcRegType> 4817: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 4818 asm, "\t$Zd, $Rn, $Rm", 4819 "", []>, Sched<[]> { 4820 bits<5> Zd; 4821 bits<5> Rm; 4822 bits<5> Rn; 4823 let Inst{31-24} = 0b00000100; 4824 let Inst{23-22} = sz8_64; 4825 let Inst{21} = 0b1; 4826 let Inst{20-16} = Rm; 4827 let Inst{15-10} = 0b010011; 4828 let Inst{9-5} = Rn; 4829 let Inst{4-0} = Zd; 4830} 4831 4832multiclass sve_int_index_rr<string asm, SDPatternOperator op> { 4833 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 4834 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 4835 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 4836 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 4837 4838 def : SVE_2_Op_Pat<nxv16i8, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4839 def : SVE_2_Op_Pat<nxv8i16, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4840 def : SVE_2_Op_Pat<nxv4i32, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4841 def : SVE_2_Op_Pat<nxv2i64, op, i64, i64, !cast<Instruction>(NAME # _D)>; 4842} 4843 4844//===----------------------------------------------------------------------===// 4845// SVE Bitwise Shift - Predicated Group 4846//===----------------------------------------------------------------------===// 4847 4848class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 4849 ZPRRegOp zprty, Operand immtype> 4850: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 4851 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 4852 "", 4853 []>, Sched<[]> { 4854 bits<3> Pg; 4855 bits<5> Zdn; 4856 bits<6> imm; 4857 let Inst{31-24} = 0b00000100; 4858 let Inst{23-22} = tsz8_64{3-2}; 4859 let Inst{21-20} = 0b00; 4860 let Inst{19-16} = opc; 4861 let Inst{15-13} = 0b100; 4862 let Inst{12-10} = Pg; 4863 let Inst{9-8} = tsz8_64{1-0}; 4864 let Inst{7-5} = imm{2-0}; // imm3 4865 let Inst{4-0} = Zdn; 4866 4867 let Constraints = "$Zdn = $_Zdn"; 4868 let DestructiveInstType = DestructiveBinaryImm; 4869 let ElementSize = zprty.ElementSize; 4870} 4871 4872multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 4873 SDPatternOperator op = null_frag> { 4874 def _B : SVEPseudo2Instr<Ps # _B, 1>, 4875 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 4876 def _H : SVEPseudo2Instr<Ps # _H, 1>, 4877 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 4878 let Inst{8} = imm{3}; 4879 } 4880 def _S : SVEPseudo2Instr<Ps # _S, 1>, 4881 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 4882 let Inst{9-8} = imm{4-3}; 4883 } 4884 def _D : SVEPseudo2Instr<Ps # _D, 1>, 4885 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 4886 let Inst{22} = imm{5}; 4887 let Inst{9-8} = imm{4-3}; 4888 } 4889 4890 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 4891 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 4892 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 4893 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 4894} 4895 4896// As above but shift amount takes the form of a "vector immediate". 4897multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 4898 string Ps, SDPatternOperator op> 4899: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 4900 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 4901 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 4902 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 4903 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 4904} 4905 4906multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 4907 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 4908 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 4909 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 4910 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 4911 4912 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 4913 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 4914 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 4915 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 4916} 4917 4918multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 4919 SDPatternOperator op = null_frag> { 4920 def _B : SVEPseudo2Instr<Ps # _B, 1>, 4921 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 4922 def _H : SVEPseudo2Instr<Ps # _H, 1>, 4923 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 4924 let Inst{8} = imm{3}; 4925 } 4926 def _S : SVEPseudo2Instr<Ps # _S, 1>, 4927 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 4928 let Inst{9-8} = imm{4-3}; 4929 } 4930 def _D : SVEPseudo2Instr<Ps # _D, 1>, 4931 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 4932 let Inst{22} = imm{5}; 4933 let Inst{9-8} = imm{4-3}; 4934 } 4935 4936 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4937 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4938 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4939 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4940} 4941 4942// As above but shift amount takes the form of a "vector immediate". 4943multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 4944 string Ps, SDPatternOperator op> 4945: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 4946 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 4947 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 4948 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 4949 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 4950} 4951 4952multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 4953 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 4954 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 4955 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 4956 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 4957 4958 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 4959 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 4960 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 4961 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 4962} 4963 4964class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 4965 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 4966: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 4967 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 4968 "", 4969 []>, Sched<[]> { 4970 bits<3> Pg; 4971 bits<5> Zdn; 4972 bits<5> Zm; 4973 let Inst{31-24} = 0b00000100; 4974 let Inst{23-22} = sz8_64; 4975 let Inst{21-20} = 0b01; 4976 let Inst{19} = wide; 4977 let Inst{18-16} = opc; 4978 let Inst{15-13} = 0b100; 4979 let Inst{12-10} = Pg; 4980 let Inst{9-5} = Zm; 4981 let Inst{4-0} = Zdn; 4982 4983 let Constraints = "$Zdn = $_Zdn"; 4984 let DestructiveInstType = DestructiveOther; 4985 let ElementSize = zprty.ElementSize; 4986} 4987 4988multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 4989 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 4990 let DestructiveInstType = DestructiveBinaryCommWithRev in { 4991 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 4992 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 4993 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 4994 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 4995 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 4996 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 4997 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 4998 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 4999 } 5000 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5001 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5002 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5003 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5004} 5005 5006multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5007 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5008 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5009 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5010 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5011 5012 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5013 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5014 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5015 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5016} 5017 5018multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5019 SDPatternOperator op> { 5020 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5021 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5022 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5023 5024 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5025 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5026 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5027} 5028 5029//===----------------------------------------------------------------------===// 5030// SVE Shift - Unpredicated Group 5031//===----------------------------------------------------------------------===// 5032 5033class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5034 ZPRRegOp zprty> 5035: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5036 asm, "\t$Zd, $Zn, $Zm", 5037 "", 5038 []>, Sched<[]> { 5039 bits<5> Zd; 5040 bits<5> Zm; 5041 bits<5> Zn; 5042 let Inst{31-24} = 0b00000100; 5043 let Inst{23-22} = sz8_64; 5044 let Inst{21} = 0b1; 5045 let Inst{20-16} = Zm; 5046 let Inst{15-12} = 0b1000; 5047 let Inst{11-10} = opc; 5048 let Inst{9-5} = Zn; 5049 let Inst{4-0} = Zd; 5050} 5051 5052multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm> { 5053 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5054 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5055 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5056} 5057 5058class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5059 ZPRRegOp zprty, Operand immtype> 5060: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5061 asm, "\t$Zd, $Zn, $imm", 5062 "", 5063 []>, Sched<[]> { 5064 bits<5> Zd; 5065 bits<5> Zn; 5066 bits<6> imm; 5067 let Inst{31-24} = 0b00000100; 5068 let Inst{23-22} = tsz8_64{3-2}; 5069 let Inst{21} = 0b1; 5070 let Inst{20-19} = tsz8_64{1-0}; 5071 let Inst{18-16} = imm{2-0}; // imm3 5072 let Inst{15-12} = 0b1001; 5073 let Inst{11-10} = opc; 5074 let Inst{9-5} = Zn; 5075 let Inst{4-0} = Zd; 5076} 5077 5078multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5079 SDPatternOperator op> { 5080 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5081 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5082 let Inst{19} = imm{3}; 5083 } 5084 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5085 let Inst{20-19} = imm{4-3}; 5086 } 5087 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5088 let Inst{22} = imm{5}; 5089 let Inst{20-19} = imm{4-3}; 5090 } 5091 5092 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5093 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5094 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5095 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5096} 5097 5098multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5099 SDPatternOperator op> { 5100 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5101 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5102 let Inst{19} = imm{3}; 5103 } 5104 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5105 let Inst{20-19} = imm{4-3}; 5106 } 5107 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5108 let Inst{22} = imm{5}; 5109 let Inst{20-19} = imm{4-3}; 5110 } 5111 5112 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5113 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5114 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5115 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5116} 5117 5118//===----------------------------------------------------------------------===// 5119// SVE Memory - Store Group 5120//===----------------------------------------------------------------------===// 5121 5122class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5123 RegisterOperand VecList> 5124: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5125 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5126 "", 5127 []>, Sched<[]> { 5128 bits<3> Pg; 5129 bits<5> Rn; 5130 bits<5> Zt; 5131 bits<4> imm4; 5132 let Inst{31-25} = 0b1110010; 5133 let Inst{24-23} = msz; 5134 let Inst{22-21} = esz; 5135 let Inst{20} = 0; 5136 let Inst{19-16} = imm4; 5137 let Inst{15-13} = 0b111; 5138 let Inst{12-10} = Pg; 5139 let Inst{9-5} = Rn; 5140 let Inst{4-0} = Zt; 5141 5142 let mayStore = 1; 5143} 5144 5145multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5146 RegisterOperand listty, ZPRRegOp zprty> 5147{ 5148 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5149 5150 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5151 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5152 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5153 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5154 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5155 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5156} 5157 5158class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5159 string asm, Operand immtype> 5160: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5161 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5162 "", 5163 []>, Sched<[]> { 5164 bits<3> Pg; 5165 bits<5> Rn; 5166 bits<5> Zt; 5167 bits<4> imm4; 5168 let Inst{31-25} = 0b1110010; 5169 let Inst{24-23} = sz; 5170 let Inst{22-21} = nregs; 5171 let Inst{20} = 1; 5172 let Inst{19-16} = imm4; 5173 let Inst{15-13} = 0b111; 5174 let Inst{12-10} = Pg; 5175 let Inst{9-5} = Rn; 5176 let Inst{4-0} = Zt; 5177 5178 let mayStore = 1; 5179} 5180 5181multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5182 string asm, Operand immtype> { 5183 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5184 5185 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5186 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5187} 5188 5189class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5190 string asm, RegisterOperand gprty> 5191: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5192 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5193 "", 5194 []>, Sched<[]> { 5195 bits<3> Pg; 5196 bits<5> Rm; 5197 bits<5> Rn; 5198 bits<5> Zt; 5199 let Inst{31-25} = 0b1110010; 5200 let Inst{24-23} = sz; 5201 let Inst{22-21} = nregs; 5202 let Inst{20-16} = Rm; 5203 let Inst{15-13} = 0b011; 5204 let Inst{12-10} = Pg; 5205 let Inst{9-5} = Rn; 5206 let Inst{4-0} = Zt; 5207 5208 let mayStore = 1; 5209} 5210 5211class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5212 RegisterOperand listty, RegisterOperand gprty> 5213: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5214 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5215 "", 5216 []>, Sched<[]> { 5217 bits<3> Pg; 5218 bits<5> Rm; 5219 bits<5> Rn; 5220 bits<5> Zt; 5221 let Inst{31-25} = 0b1110010; 5222 let Inst{24-21} = dtype; 5223 let Inst{20-16} = Rm; 5224 let Inst{15-13} = 0b010; 5225 let Inst{12-10} = Pg; 5226 let Inst{9-5} = Rn; 5227 let Inst{4-0} = Zt; 5228 5229 let mayStore = 1; 5230} 5231 5232multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5233 RegisterOperand listty, ZPRRegOp zprty, 5234 RegisterOperand gprty> { 5235 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5236 5237 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5238 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5239} 5240 5241class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5242: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5243 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5244 "", 5245 []>, Sched<[]> { 5246 bits<3> Pg; 5247 bits<5> Rn; 5248 bits<5> Zt; 5249 bits<4> imm4; 5250 let Inst{31-25} = 0b1110010; 5251 let Inst{24-23} = msz; 5252 let Inst{22-20} = 0b001; 5253 let Inst{19-16} = imm4; 5254 let Inst{15-13} = 0b111; 5255 let Inst{12-10} = Pg; 5256 let Inst{9-5} = Rn; 5257 let Inst{4-0} = Zt; 5258 5259 let mayStore = 1; 5260} 5261 5262multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5263 ZPRRegOp zprty> { 5264 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5265 5266 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5267 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5268 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5269 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5270 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5271 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5272} 5273 5274class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5275 RegisterOperand gprty> 5276: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5277 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5278 "", 5279 []>, Sched<[]> { 5280 bits<3> Pg; 5281 bits<5> Rm; 5282 bits<5> Rn; 5283 bits<5> Zt; 5284 let Inst{31-25} = 0b1110010; 5285 let Inst{24-23} = msz; 5286 let Inst{22-21} = 0b00; 5287 let Inst{20-16} = Rm; 5288 let Inst{15-13} = 0b011; 5289 let Inst{12-10} = Pg; 5290 let Inst{9-5} = Rn; 5291 let Inst{4-0} = Zt; 5292 5293 let mayStore = 1; 5294} 5295 5296multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5297 ZPRRegOp zprty, RegisterOperand gprty> { 5298 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5299 5300 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5301 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5302} 5303 5304class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5305 RegisterOperand listty, ZPRRegOp zprty> 5306: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5307 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5308 "", 5309 []>, Sched<[]> { 5310 bits<3> Pg; 5311 bits<5> Rm; 5312 bits<5> Zn; 5313 bits<5> Zt; 5314 let Inst{31-25} = 0b1110010; 5315 let Inst{24-22} = opc; 5316 let Inst{21} = 0b0; 5317 let Inst{20-16} = Rm; 5318 let Inst{15-13} = 0b001; 5319 let Inst{12-10} = Pg; 5320 let Inst{9-5} = Zn; 5321 let Inst{4-0} = Zt; 5322 5323 let mayStore = 1; 5324} 5325 5326multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5327 SDPatternOperator op, 5328 ValueType vt> { 5329 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5330 5331 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5332 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5333 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5334 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5335 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5336 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5337 5338 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5339 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5340} 5341 5342multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5343 SDPatternOperator op, 5344 ValueType vt> { 5345 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5346 5347 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5348 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5349 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5350 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5351 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5352 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5353 5354 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5355 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5356} 5357 5358class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5359 RegisterOperand VecList, RegisterOperand zprext> 5360: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5361 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5362 "", 5363 []>, Sched<[]> { 5364 bits<3> Pg; 5365 bits<5> Rn; 5366 bits<5> Zm; 5367 bits<5> Zt; 5368 let Inst{31-25} = 0b1110010; 5369 let Inst{24-22} = opc; 5370 let Inst{21} = scaled; 5371 let Inst{20-16} = Zm; 5372 let Inst{15} = 0b1; 5373 let Inst{14} = xs; 5374 let Inst{13} = 0; 5375 let Inst{12-10} = Pg; 5376 let Inst{9-5} = Rn; 5377 let Inst{4-0} = Zt; 5378 5379 let mayStore = 1; 5380} 5381 5382multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5383 SDPatternOperator sxtw_op, 5384 SDPatternOperator uxtw_op, 5385 RegisterOperand sxtw_opnd, 5386 RegisterOperand uxtw_opnd, 5387 ValueType vt > { 5388 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5389 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5390 5391 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5392 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5393 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5394 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5395 5396 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5397 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5398 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5399 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5400} 5401 5402multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5403 SDPatternOperator sxtw_op, 5404 SDPatternOperator uxtw_op, 5405 RegisterOperand sxtw_opnd, 5406 RegisterOperand uxtw_opnd, 5407 ValueType vt > { 5408 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 5409 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 5410 5411 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5412 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5413 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5414 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5415 5416 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5417 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5418 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5419 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5420} 5421 5422multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 5423 SDPatternOperator sxtw_op, 5424 SDPatternOperator uxtw_op, 5425 RegisterOperand sxtw_opnd, 5426 RegisterOperand uxtw_opnd, 5427 ValueType vt> { 5428 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 5429 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 5430 5431 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5432 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5433 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5434 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5435 5436 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5437 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5438 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5439 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5440} 5441 5442multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 5443 SDPatternOperator sxtw_op, 5444 SDPatternOperator uxtw_op, 5445 RegisterOperand sxtw_opnd, 5446 RegisterOperand uxtw_opnd, 5447 ValueType vt> { 5448 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 5449 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 5450 5451 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5452 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5453 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5454 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5455 5456 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5457 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5458 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5459 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5460} 5461 5462class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 5463 RegisterOperand zprext> 5464: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5465 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5466 "", 5467 []>, Sched<[]> { 5468 bits<3> Pg; 5469 bits<5> Rn; 5470 bits<5> Zm; 5471 bits<5> Zt; 5472 let Inst{31-25} = 0b1110010; 5473 let Inst{24-23} = msz; 5474 let Inst{22} = 0b0; 5475 let Inst{21} = scaled; 5476 let Inst{20-16} = Zm; 5477 let Inst{15-13} = 0b101; 5478 let Inst{12-10} = Pg; 5479 let Inst{9-5} = Rn; 5480 let Inst{4-0} = Zt; 5481 5482 let mayStore = 1; 5483} 5484 5485multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 5486 SDPatternOperator op, 5487 RegisterOperand zprext, 5488 ValueType vt> { 5489 def _SCALED_REAL : sve_mem_sst_sv2<msz, 1, asm, zprext>; 5490 5491 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5492 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 5493 5494 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 5495 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 5496} 5497 5498multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 5499 SDPatternOperator op, 5500 ValueType vt> { 5501 def _REAL : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 5502 5503 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5504 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 5505 5506 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5507 (!cast<Instruction>(NAME # _REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5508} 5509 5510class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 5511 RegisterOperand VecList, Operand imm_ty> 5512: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 5513 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 5514 "", 5515 []>, Sched<[]> { 5516 bits<3> Pg; 5517 bits<5> imm5; 5518 bits<5> Zn; 5519 bits<5> Zt; 5520 let Inst{31-25} = 0b1110010; 5521 let Inst{24-23} = opc{2-1}; 5522 let Inst{22} = 0b1; 5523 let Inst{21} = opc{0}; 5524 let Inst{20-16} = imm5; 5525 let Inst{15-13} = 0b101; 5526 let Inst{12-10} = Pg; 5527 let Inst{9-5} = Zn; 5528 let Inst{4-0} = Zt; 5529 5530 let mayStore = 1; 5531} 5532 5533multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 5534 Operand imm_ty, 5535 SDPatternOperator op, 5536 ValueType vt> { 5537 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 5538 5539 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5540 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 5541 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5542 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 5543 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5544 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 5545 5546 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 5547 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5548} 5549 5550multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 5551 Operand imm_ty, 5552 SDPatternOperator op, 5553 ValueType vt> { 5554 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 5555 5556 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5557 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 5558 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5559 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 5560 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5561 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 5562 5563 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 5564 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5565} 5566 5567class sve_mem_z_spill<string asm> 5568: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 5569 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 5570 "", 5571 []>, Sched<[]> { 5572 bits<5> Rn; 5573 bits<5> Zt; 5574 bits<9> imm9; 5575 let Inst{31-22} = 0b1110010110; 5576 let Inst{21-16} = imm9{8-3}; 5577 let Inst{15-13} = 0b010; 5578 let Inst{12-10} = imm9{2-0}; 5579 let Inst{9-5} = Rn; 5580 let Inst{4-0} = Zt; 5581 5582 let mayStore = 1; 5583} 5584 5585multiclass sve_mem_z_spill<string asm> { 5586 def NAME : sve_mem_z_spill<asm>; 5587 5588 def : InstAlias<asm # "\t$Zt, [$Rn]", 5589 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 5590} 5591 5592class sve_mem_p_spill<string asm> 5593: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 5594 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 5595 "", 5596 []>, Sched<[]> { 5597 bits<4> Pt; 5598 bits<5> Rn; 5599 bits<9> imm9; 5600 let Inst{31-22} = 0b1110010110; 5601 let Inst{21-16} = imm9{8-3}; 5602 let Inst{15-13} = 0b000; 5603 let Inst{12-10} = imm9{2-0}; 5604 let Inst{9-5} = Rn; 5605 let Inst{4} = 0b0; 5606 let Inst{3-0} = Pt; 5607 5608 let mayStore = 1; 5609} 5610 5611multiclass sve_mem_p_spill<string asm> { 5612 def NAME : sve_mem_p_spill<asm>; 5613 5614 def : InstAlias<asm # "\t$Pt, [$Rn]", 5615 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 5616} 5617 5618//===----------------------------------------------------------------------===// 5619// SVE Permute - Predicates Group 5620//===----------------------------------------------------------------------===// 5621 5622class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 5623 PPRRegOp pprty> 5624: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 5625 asm, "\t$Pd, $Pn, $Pm", 5626 "", []>, Sched<[]> { 5627 bits<4> Pd; 5628 bits<4> Pm; 5629 bits<4> Pn; 5630 let Inst{31-24} = 0b00000101; 5631 let Inst{23-22} = sz8_64; 5632 let Inst{21-20} = 0b10; 5633 let Inst{19-16} = Pm; 5634 let Inst{15-13} = 0b010; 5635 let Inst{12-10} = opc; 5636 let Inst{9} = 0b0; 5637 let Inst{8-5} = Pn; 5638 let Inst{4} = 0b0; 5639 let Inst{3-0} = Pd; 5640} 5641 5642multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 5643 SDPatternOperator op> { 5644 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 5645 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 5646 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 5647 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 5648 5649 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 5650 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 5651 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 5652 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 5653} 5654 5655class sve_int_perm_punpk<bit opc, string asm> 5656: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 5657 asm, "\t$Pd, $Pn", 5658 "", 5659 []>, Sched<[]> { 5660 bits<4> Pd; 5661 bits<4> Pn; 5662 let Inst{31-17} = 0b000001010011000; 5663 let Inst{16} = opc; 5664 let Inst{15-9} = 0b0100000; 5665 let Inst{8-5} = Pn; 5666 let Inst{4} = 0b0; 5667 let Inst{3-0} = Pd; 5668} 5669 5670multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 5671 def NAME : sve_int_perm_punpk<opc, asm>; 5672 5673 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 5674 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 5675 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 5676} 5677 5678class sve_int_rdffr_pred<bit s, string asm> 5679: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 5680 asm, "\t$Pd, $Pg/z", 5681 "", 5682 []>, Sched<[]> { 5683 bits<4> Pd; 5684 bits<4> Pg; 5685 let Inst{31-23} = 0b001001010; 5686 let Inst{22} = s; 5687 let Inst{21-9} = 0b0110001111000; 5688 let Inst{8-5} = Pg; 5689 let Inst{4} = 0; 5690 let Inst{3-0} = Pd; 5691 5692 let Defs = !if(s, [NZCV], []); 5693 let Uses = [FFR]; 5694} 5695 5696multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 5697 def _REAL : sve_int_rdffr_pred<s, asm>; 5698 5699 // We need a layer of indirection because early machine code passes balk at 5700 // physical register (i.e. FFR) uses that have no previous definition. 5701 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 5702 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 5703 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 5704 } 5705} 5706 5707class sve_int_rdffr_unpred<string asm> : I< 5708 (outs PPR8:$Pd), (ins), 5709 asm, "\t$Pd", 5710 "", 5711 []>, Sched<[]> { 5712 bits<4> Pd; 5713 let Inst{31-4} = 0b0010010100011001111100000000; 5714 let Inst{3-0} = Pd; 5715 5716 let Uses = [FFR]; 5717} 5718 5719multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 5720 def _REAL : sve_int_rdffr_unpred<asm>; 5721 5722 // We need a layer of indirection because early machine code passes balk at 5723 // physical register (i.e. FFR) uses that have no previous definition. 5724 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 5725 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 5726 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 5727 } 5728} 5729 5730class sve_int_wrffr<string asm, SDPatternOperator op> 5731: I<(outs), (ins PPR8:$Pn), 5732 asm, "\t$Pn", 5733 "", 5734 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 5735 bits<4> Pn; 5736 let Inst{31-9} = 0b00100101001010001001000; 5737 let Inst{8-5} = Pn; 5738 let Inst{4-0} = 0b00000; 5739 5740 let hasSideEffects = 1; 5741 let Defs = [FFR]; 5742} 5743 5744class sve_int_setffr<string asm, SDPatternOperator op> 5745: I<(outs), (ins), 5746 asm, "", 5747 "", 5748 [(op)]>, Sched<[]> { 5749 let Inst{31-0} = 0b00100101001011001001000000000000; 5750 5751 let hasSideEffects = 1; 5752 let Defs = [FFR]; 5753} 5754 5755//===----------------------------------------------------------------------===// 5756// SVE Permute Vector - Predicated Group 5757//===----------------------------------------------------------------------===// 5758 5759class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 5760 ZPRRegOp zprty, RegisterClass rt> 5761: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 5762 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 5763 "", 5764 []>, Sched<[]> { 5765 bits<3> Pg; 5766 bits<5> Rdn; 5767 bits<5> Zm; 5768 let Inst{31-24} = 0b00000101; 5769 let Inst{23-22} = sz8_64; 5770 let Inst{21-17} = 0b11000; 5771 let Inst{16} = ab; 5772 let Inst{15-13} = 0b101; 5773 let Inst{12-10} = Pg; 5774 let Inst{9-5} = Zm; 5775 let Inst{4-0} = Rdn; 5776 5777 let Constraints = "$Rdn = $_Rdn"; 5778} 5779 5780multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 5781 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 5782 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 5783 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 5784 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 5785 5786 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 5787 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 5788 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5789 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5790} 5791 5792class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 5793 ZPRRegOp zprty, RegisterClass rt> 5794: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 5795 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 5796 "", 5797 []>, Sched<[]> { 5798 bits<3> Pg; 5799 bits<5> Vdn; 5800 bits<5> Zm; 5801 let Inst{31-24} = 0b00000101; 5802 let Inst{23-22} = sz8_64; 5803 let Inst{21-17} = 0b10101; 5804 let Inst{16} = ab; 5805 let Inst{15-13} = 0b100; 5806 let Inst{12-10} = Pg; 5807 let Inst{9-5} = Zm; 5808 let Inst{4-0} = Vdn; 5809 5810 let Constraints = "$Vdn = $_Vdn"; 5811} 5812 5813multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 5814 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 5815 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 5816 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 5817 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 5818 5819 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5820 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5821 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5822 5823 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 5824} 5825 5826class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 5827 ZPRRegOp zprty> 5828: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 5829 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 5830 "", 5831 []>, Sched<[]> { 5832 bits<3> Pg; 5833 bits<5> Zdn; 5834 bits<5> Zm; 5835 let Inst{31-24} = 0b00000101; 5836 let Inst{23-22} = sz8_64; 5837 let Inst{21-17} = 0b10100; 5838 let Inst{16} = ab; 5839 let Inst{15-13} = 0b100; 5840 let Inst{12-10} = Pg; 5841 let Inst{9-5} = Zm; 5842 let Inst{4-0} = Zdn; 5843 5844 let Constraints = "$Zdn = $_Zdn"; 5845 let DestructiveInstType = DestructiveOther; 5846 let ElementSize = ElementSizeNone; 5847} 5848 5849multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 5850 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 5851 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 5852 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 5853 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 5854 5855 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5856 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5857 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5858 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5859 5860 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5861 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5862 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5863 5864 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 5865} 5866 5867class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 5868 ZPRRegOp zprty, RegisterClass resultRegType> 5869: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 5870 asm, "\t$Rd, $Pg, $Zn", 5871 "", 5872 []>, Sched<[]> { 5873 bits<3> Pg; 5874 bits<5> Rd; 5875 bits<5> Zn; 5876 let Inst{31-24} = 0b00000101; 5877 let Inst{23-22} = sz8_64; 5878 let Inst{21-17} = 0b10000; 5879 let Inst{16} = ab; 5880 let Inst{15-13} = 0b101; 5881 let Inst{12-10} = Pg; 5882 let Inst{9-5} = Zn; 5883 let Inst{4-0} = Rd; 5884} 5885 5886multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 5887 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 5888 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 5889 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 5890 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 5891 5892 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 5893 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 5894 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 5895 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 5896} 5897 5898class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 5899 ZPRRegOp zprty, RegisterClass dstRegtype> 5900: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 5901 asm, "\t$Vd, $Pg, $Zn", 5902 "", 5903 []>, Sched<[]> { 5904 bits<3> Pg; 5905 bits<5> Vd; 5906 bits<5> Zn; 5907 let Inst{31-24} = 0b00000101; 5908 let Inst{23-22} = sz8_64; 5909 let Inst{21-17} = 0b10001; 5910 let Inst{16} = ab; 5911 let Inst{15-13} = 0b100; 5912 let Inst{12-10} = Pg; 5913 let Inst{9-5} = Zn; 5914 let Inst{4-0} = Vd; 5915} 5916 5917multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 5918 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 5919 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 5920 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 5921 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 5922 5923 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5924 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5925 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5926 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5927 5928 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 5929} 5930 5931class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 5932: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 5933 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 5934 "", 5935 []>, Sched<[]> { 5936 bits<3> Pg; 5937 bits<5> Zdn; 5938 bits<5> Zm; 5939 let Inst{31-24} = 0b00000101; 5940 let Inst{23-22} = sz8_64; 5941 let Inst{21-13} = 0b101100100; 5942 let Inst{12-10} = Pg; 5943 let Inst{9-5} = Zm; 5944 let Inst{4-0} = Zdn; 5945 5946 let Constraints = "$Zdn = $_Zdn"; 5947 let DestructiveInstType = DestructiveOther; 5948 let ElementSize = ElementSizeNone; 5949} 5950 5951multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 5952 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 5953 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 5954 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 5955 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 5956 5957 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5958 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5959 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5960 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5961 5962 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5963 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5964 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5965 5966 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 5967} 5968 5969class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 5970 ZPRRegOp zprty, RegisterOperand VecList> 5971: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 5972 asm, "\t$Zd, $Pg, $Zn", 5973 "", 5974 []>, Sched<[]> { 5975 bits<3> Pg; 5976 bits<5> Zn; 5977 bits<5> Zd; 5978 let Inst{31-24} = 0b00000101; 5979 let Inst{23-22} = sz8_64; 5980 let Inst{21-13} = 0b101101100; 5981 let Inst{12-10} = Pg; 5982 let Inst{9-5} = Zn; 5983 let Inst{4-0} = Zd; 5984} 5985 5986multiclass sve2_int_perm_splice_cons<string asm> { 5987 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 5988 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 5989 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 5990 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 5991} 5992 5993class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 5994 ZPRRegOp zprty> 5995: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 5996 asm, "\t$Zd, $Pg/m, $Zn", 5997 "", 5998 []>, Sched<[]> { 5999 bits<5> Zd; 6000 bits<3> Pg; 6001 bits<5> Zn; 6002 let Inst{31-24} = 0b00000101; 6003 let Inst{23-22} = sz8_64; 6004 let Inst{21-18} = 0b1001; 6005 let Inst{17-16} = opc; 6006 let Inst{15-13} = 0b100; 6007 let Inst{12-10} = Pg; 6008 let Inst{9-5} = Zn; 6009 let Inst{4-0} = Zd; 6010 6011 let Constraints = "$Zd = $_Zd"; 6012 let DestructiveInstType = DestructiveOther; 6013 let ElementSize = zprty.ElementSize; 6014} 6015 6016multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6017 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6018 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6019 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6020 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6021 6022 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6023 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6024 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6025 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6026} 6027 6028multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6029 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6030 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6031 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6032 6033 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6034 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6035 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6036} 6037 6038multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6039 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6040 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6041 6042 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6043 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6044} 6045 6046multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6047 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6048 6049 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6050} 6051 6052class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6053 RegisterClass srcRegType> 6054: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6055 asm, "\t$Zd, $Pg/m, $Rn", 6056 "", 6057 []>, Sched<[]> { 6058 bits<3> Pg; 6059 bits<5> Rn; 6060 bits<5> Zd; 6061 let Inst{31-24} = 0b00000101; 6062 let Inst{23-22} = sz8_64; 6063 let Inst{21-13} = 0b101000101; 6064 let Inst{12-10} = Pg; 6065 let Inst{9-5} = Rn; 6066 let Inst{4-0} = Zd; 6067 6068 let Constraints = "$Zd = $_Zd"; 6069 let DestructiveInstType = DestructiveOther; 6070 let ElementSize = zprty.ElementSize; 6071} 6072 6073multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6074 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6075 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6076 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6077 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6078 6079 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6080 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6081 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6082 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6083 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6084 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6085 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6086 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6087 6088 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6089 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6090 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6091 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6092 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6093 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6094 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6095 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6096} 6097 6098class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6099 RegisterClass srcRegtype> 6100: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6101 asm, "\t$Zd, $Pg/m, $Vn", 6102 "", 6103 []>, Sched<[]> { 6104 bits<3> Pg; 6105 bits<5> Vn; 6106 bits<5> Zd; 6107 let Inst{31-24} = 0b00000101; 6108 let Inst{23-22} = sz8_64; 6109 let Inst{21-13} = 0b100000100; 6110 let Inst{12-10} = Pg; 6111 let Inst{9-5} = Vn; 6112 let Inst{4-0} = Zd; 6113 6114 let Constraints = "$Zd = $_Zd"; 6115 let DestructiveInstType = DestructiveOther; 6116 let ElementSize = zprty.ElementSize; 6117} 6118 6119multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6120 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6121 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6122 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6123 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6124 6125 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6126 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6127 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6128 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6129 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6130 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6131 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6132 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6133 6134 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6135 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6136 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6137 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6138 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6139 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6140 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6141 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6142 6143 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6144 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6145} 6146 6147class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6148: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6149 asm, "\t$Zd, $Pg, $Zn", 6150 "", 6151 []>, Sched<[]> { 6152 bits<3> Pg; 6153 bits<5> Zd; 6154 bits<5> Zn; 6155 let Inst{31-23} = 0b000001011; 6156 let Inst{22} = sz; 6157 let Inst{21-13} = 0b100001100; 6158 let Inst{12-10} = Pg; 6159 let Inst{9-5} = Zn; 6160 let Inst{4-0} = Zd; 6161} 6162 6163multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6164 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6165 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6166 6167 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6168 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6169 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6170 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6171} 6172 6173//===----------------------------------------------------------------------===// 6174// SVE Memory - Contiguous Load Group 6175//===----------------------------------------------------------------------===// 6176 6177class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6178 RegisterOperand VecList> 6179: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6180 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6181 "", 6182 []>, Sched<[]> { 6183 bits<3> Pg; 6184 bits<5> Rn; 6185 bits<5> Zt; 6186 bits<4> imm4; 6187 let Inst{31-25} = 0b1010010; 6188 let Inst{24-21} = dtype; 6189 let Inst{20} = nf; 6190 let Inst{19-16} = imm4; 6191 let Inst{15-13} = 0b101; 6192 let Inst{12-10} = Pg; 6193 let Inst{9-5} = Rn; 6194 let Inst{4-0} = Zt; 6195 6196 let mayLoad = 1; 6197 let Uses = !if(nf, [FFR], []); 6198 let Defs = !if(nf, [FFR], []); 6199} 6200 6201multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6202 RegisterOperand listty, ZPRRegOp zprty> { 6203 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6204 6205 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6206 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6207 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6208 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6209 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6210 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6211 6212 // We need a layer of indirection because early machine code passes balk at 6213 // physical register (i.e. FFR) uses that have no previous definition. 6214 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6215 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6216 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6217 } 6218} 6219 6220multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6221 ZPRRegOp zprty> 6222: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6223 6224class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6225: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6226 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6227 "", 6228 []>, Sched<[]> { 6229 bits<5> Zt; 6230 bits<3> Pg; 6231 bits<5> Rn; 6232 bits<4> imm4; 6233 let Inst{31-25} = 0b1010010; 6234 let Inst{24-23} = msz; 6235 let Inst{22-20} = 0b000; 6236 let Inst{19-16} = imm4; 6237 let Inst{15-13} = 0b111; 6238 let Inst{12-10} = Pg; 6239 let Inst{9-5} = Rn; 6240 let Inst{4-0} = Zt; 6241 6242 let mayLoad = 1; 6243} 6244 6245multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6246 ZPRRegOp zprty> { 6247 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6248 6249 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6250 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6251 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6252 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6253 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6254 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6255} 6256 6257class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6258 RegisterOperand gprty> 6259: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6260 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6261 "", 6262 []>, Sched<[]> { 6263 bits<3> Pg; 6264 bits<5> Rm; 6265 bits<5> Rn; 6266 bits<5> Zt; 6267 let Inst{31-25} = 0b1010010; 6268 let Inst{24-23} = msz; 6269 let Inst{22-21} = 0b00; 6270 let Inst{20-16} = Rm; 6271 let Inst{15-13} = 0b110; 6272 let Inst{12-10} = Pg; 6273 let Inst{9-5} = Rn; 6274 let Inst{4-0} = Zt; 6275 6276 let mayLoad = 1; 6277} 6278 6279multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6280 ZPRRegOp zprty, RegisterOperand gprty> { 6281 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6282 6283 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6284 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6285} 6286 6287class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6288: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6289 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6290 bits<5> Zt; 6291 bits<5> Rn; 6292 bits<3> Pg; 6293 bits<4> imm4; 6294 let Inst{31-25} = 0b1010010; 6295 let Inst{24-23} = sz; 6296 let Inst{22-20} = 0; 6297 let Inst{19-16} = imm4; 6298 let Inst{15-13} = 0b001; 6299 let Inst{12-10} = Pg; 6300 let Inst{9-5} = Rn; 6301 let Inst{4-0} = Zt; 6302 6303 let mayLoad = 1; 6304} 6305 6306multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6307 ZPRRegOp zprty> { 6308 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6309 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6310 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6311 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6312 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6313 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6314 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6315} 6316 6317class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6318 RegisterOperand gprty> 6319: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6320 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6321 bits<5> Zt; 6322 bits<3> Pg; 6323 bits<5> Rn; 6324 bits<5> Rm; 6325 let Inst{31-25} = 0b1010010; 6326 let Inst{24-23} = sz; 6327 let Inst{22-21} = 0; 6328 let Inst{20-16} = Rm; 6329 let Inst{15-13} = 0; 6330 let Inst{12-10} = Pg; 6331 let Inst{9-5} = Rn; 6332 let Inst{4-0} = Zt; 6333 6334 let mayLoad = 1; 6335} 6336 6337multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6338 ZPRRegOp zprty, RegisterOperand gprty> { 6339 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6340 6341 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6342 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6343} 6344 6345class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6346 RegisterOperand VecList, Operand immtype> 6347: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6348 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6349 "", 6350 []>, Sched<[]> { 6351 bits<3> Pg; 6352 bits<5> Rn; 6353 bits<5> Zt; 6354 bits<6> imm6; 6355 let Inst{31-25} = 0b1000010; 6356 let Inst{24-23} = dtypeh; 6357 let Inst{22} = 1; 6358 let Inst{21-16} = imm6; 6359 let Inst{15} = 0b1; 6360 let Inst{14-13} = dtypel; 6361 let Inst{12-10} = Pg; 6362 let Inst{9-5} = Rn; 6363 let Inst{4-0} = Zt; 6364 6365 let mayLoad = 1; 6366} 6367 6368multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6369 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6370 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6371 6372 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6373 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6374 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6375 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6376 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6377 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6378} 6379 6380class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6381 RegisterOperand VecList> 6382: I<(outs VecList:$Zt), iops, 6383 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6384 "", 6385 []>, Sched<[]> { 6386 bits<5> Zt; 6387 bits<3> Pg; 6388 bits<5> Rm; 6389 bits<5> Rn; 6390 let Inst{31-25} = 0b1010010; 6391 let Inst{24-21} = dtype; 6392 let Inst{20-16} = Rm; 6393 let Inst{15-14} = 0b01; 6394 let Inst{13} = ff; 6395 let Inst{12-10} = Pg; 6396 let Inst{9-5} = Rn; 6397 let Inst{4-0} = Zt; 6398 6399 let mayLoad = 1; 6400 let Uses = !if(ff, [FFR], []); 6401 let Defs = !if(ff, [FFR], []); 6402} 6403 6404multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6405 ZPRRegOp zprty, RegisterOperand gprty> { 6406 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6407 asm, listty>; 6408 6409 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6410 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6411} 6412 6413multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 6414 ZPRRegOp zprty, RegisterOperand gprty> { 6415 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6416 asm, listty>; 6417 6418 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6419 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6420 6421 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6422 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 6423 6424 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6425 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 6426 6427 // We need a layer of indirection because early machine code passes balk at 6428 // physical register (i.e. FFR) uses that have no previous definition. 6429 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6430 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 6431 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 6432 } 6433} 6434 6435multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 6436 ZPRRegOp zprty> 6437: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 6438 6439class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6440 string asm, Operand immtype> 6441: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 6442 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6443 "", 6444 []>, Sched<[]> { 6445 bits<5> Zt; 6446 bits<3> Pg; 6447 bits<5> Rn; 6448 bits<4> imm4; 6449 let Inst{31-25} = 0b1010010; 6450 let Inst{24-23} = sz; 6451 let Inst{22-21} = nregs; 6452 let Inst{20} = 0; 6453 let Inst{19-16} = imm4; 6454 let Inst{15-13} = 0b111; 6455 let Inst{12-10} = Pg; 6456 let Inst{9-5} = Rn; 6457 let Inst{4-0} = Zt; 6458 6459 let mayLoad = 1; 6460} 6461 6462multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6463 string asm, Operand immtype> { 6464 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 6465 6466 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6467 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6468} 6469 6470class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6471 string asm, RegisterOperand gprty> 6472: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6473 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6474 "", 6475 []>, Sched<[]> { 6476 bits<3> Pg; 6477 bits<5> Rm; 6478 bits<5> Rn; 6479 bits<5> Zt; 6480 let Inst{31-25} = 0b1010010; 6481 let Inst{24-23} = sz; 6482 let Inst{22-21} = nregs; 6483 let Inst{20-16} = Rm; 6484 let Inst{15-13} = 0b110; 6485 let Inst{12-10} = Pg; 6486 let Inst{9-5} = Rn; 6487 let Inst{4-0} = Zt; 6488 6489 let mayLoad = 1; 6490} 6491 6492//===----------------------------------------------------------------------===// 6493// SVE Memory - 32-bit Gather and Unsized Contiguous Group 6494//===----------------------------------------------------------------------===// 6495 6496// bit xs is '1' if offsets are signed 6497// bit scaled is '1' if the offsets are scaled 6498class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 6499 RegisterOperand zprext> 6500: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6501 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 6502 "", 6503 []>, Sched<[]> { 6504 bits<3> Pg; 6505 bits<5> Rn; 6506 bits<5> Zm; 6507 bits<5> Zt; 6508 let Inst{31-25} = 0b1000010; 6509 let Inst{24-23} = opc{3-2}; 6510 let Inst{22} = xs; 6511 let Inst{21} = scaled; 6512 let Inst{20-16} = Zm; 6513 let Inst{15} = 0b0; 6514 let Inst{14-13} = opc{1-0}; 6515 let Inst{12-10} = Pg; 6516 let Inst{9-5} = Rn; 6517 let Inst{4-0} = Zt; 6518 6519 let mayLoad = 1; 6520 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6521 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6522} 6523 6524multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 6525 SDPatternOperator sxtw_op, 6526 SDPatternOperator uxtw_op, 6527 RegisterOperand sxtw_opnd, 6528 RegisterOperand uxtw_opnd, 6529 ValueType vt> { 6530 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 6531 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 6532 6533 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6534 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6535 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6536 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6537 6538 // We need a layer of indirection because early machine code passes balk at 6539 // physical register (i.e. FFR) uses that have no previous definition. 6540 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6541 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6542 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6543 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6544 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6545 } 6546 6547 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6548 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6549 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6550 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6551} 6552 6553multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 6554 SDPatternOperator sxtw_op, 6555 SDPatternOperator uxtw_op, 6556 RegisterOperand sxtw_opnd, 6557 RegisterOperand uxtw_opnd, 6558 ValueType vt> { 6559 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 6560 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 6561 6562 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6563 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6564 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6565 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6566 6567 // We need a layer of indirection because early machine code passes balk at 6568 // physical register (i.e. FFR) uses that have no previous definition. 6569 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6570 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6571 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6572 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6573 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6574 } 6575 6576 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 6577 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6578 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 6579 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6580} 6581 6582 6583class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 6584: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 6585 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 6586 "", 6587 []>, Sched<[]> { 6588 bits<3> Pg; 6589 bits<5> Zn; 6590 bits<5> Zt; 6591 bits<5> imm5; 6592 let Inst{31-25} = 0b1000010; 6593 let Inst{24-23} = opc{3-2}; 6594 let Inst{22-21} = 0b01; 6595 let Inst{20-16} = imm5; 6596 let Inst{15} = 0b1; 6597 let Inst{14-13} = opc{1-0}; 6598 let Inst{12-10} = Pg; 6599 let Inst{9-5} = Zn; 6600 let Inst{4-0} = Zt; 6601 6602 let mayLoad = 1; 6603 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6604 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6605} 6606 6607multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 6608 SDPatternOperator op, ValueType vt> { 6609 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 6610 6611 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6612 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6613 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 6614 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6615 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6616 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6617 6618 // We need a layer of indirection because early machine code passes balk at 6619 // physical register (i.e. FFR) uses that have no previous definition. 6620 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6621 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 6622 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 6623 } 6624 6625 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 6626 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6627} 6628 6629class sve_mem_prfm_si<bits<2> msz, string asm> 6630: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 6631 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 6632 "", 6633 []>, Sched<[]> { 6634 bits<5> Rn; 6635 bits<3> Pg; 6636 bits<6> imm6; 6637 bits<4> prfop; 6638 let Inst{31-22} = 0b1000010111; 6639 let Inst{21-16} = imm6; 6640 let Inst{15} = 0b0; 6641 let Inst{14-13} = msz; 6642 let Inst{12-10} = Pg; 6643 let Inst{9-5} = Rn; 6644 let Inst{4} = 0b0; 6645 let Inst{3-0} = prfop; 6646 6647 let hasSideEffects = 1; 6648} 6649 6650multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 6651 def NAME : sve_mem_prfm_si<msz, asm>; 6652 6653 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 6654 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6655} 6656 6657class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 6658: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6659 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 6660 "", 6661 []>, Sched<[]> { 6662 bits<5> Rm; 6663 bits<5> Rn; 6664 bits<3> Pg; 6665 bits<4> prfop; 6666 let Inst{31-25} = 0b1000010; 6667 let Inst{24-23} = opc{2-1}; 6668 let Inst{22-21} = 0b00; 6669 let Inst{20-16} = Rm; 6670 let Inst{15} = 0b1; 6671 let Inst{14} = opc{0}; 6672 let Inst{13} = 0b0; 6673 let Inst{12-10} = Pg; 6674 let Inst{9-5} = Rn; 6675 let Inst{4} = 0b0; 6676 let Inst{3-0} = prfop; 6677 6678 let hasSideEffects = 1; 6679} 6680 6681class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 6682 RegisterOperand zprext> 6683: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6684 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 6685 "", 6686 []>, Sched<[]> { 6687 bits<3> Pg; 6688 bits<5> Rn; 6689 bits<5> Zm; 6690 bits<4> prfop; 6691 let Inst{31-23} = 0b100001000; 6692 let Inst{22} = xs; 6693 let Inst{21} = 0b1; 6694 let Inst{20-16} = Zm; 6695 let Inst{15} = 0b0; 6696 let Inst{14-13} = msz; 6697 let Inst{12-10} = Pg; 6698 let Inst{9-5} = Rn; 6699 let Inst{4} = 0b0; 6700 let Inst{3-0} = prfop; 6701 6702 let hasSideEffects = 1; 6703} 6704 6705multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 6706 RegisterOperand sxtw_opnd, 6707 RegisterOperand uxtw_opnd, 6708 PatFrag op_sxtw, 6709 PatFrag op_uxtw> { 6710 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 6711 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 6712 6713 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 6714 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6715 6716 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 6717 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6718} 6719 6720class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 6721: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 6722 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 6723 "", 6724 []>, Sched<[]> { 6725 bits<3> Pg; 6726 bits<5> Zn; 6727 bits<5> imm5; 6728 bits<4> prfop; 6729 let Inst{31-25} = 0b1000010; 6730 let Inst{24-23} = msz; 6731 let Inst{22-21} = 0b00; 6732 let Inst{20-16} = imm5; 6733 let Inst{15-13} = 0b111; 6734 let Inst{12-10} = Pg; 6735 let Inst{9-5} = Zn; 6736 let Inst{4} = 0b0; 6737 let Inst{3-0} = prfop; 6738} 6739 6740multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 6741 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 6742 6743 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 6744 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6745 6746 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 6747 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 6748} 6749 6750class sve_mem_z_fill<string asm> 6751: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 6752 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6753 "", 6754 []>, Sched<[]> { 6755 bits<5> Rn; 6756 bits<5> Zt; 6757 bits<9> imm9; 6758 let Inst{31-22} = 0b1000010110; 6759 let Inst{21-16} = imm9{8-3}; 6760 let Inst{15-13} = 0b010; 6761 let Inst{12-10} = imm9{2-0}; 6762 let Inst{9-5} = Rn; 6763 let Inst{4-0} = Zt; 6764 6765 let mayLoad = 1; 6766} 6767 6768multiclass sve_mem_z_fill<string asm> { 6769 def NAME : sve_mem_z_fill<asm>; 6770 6771 def : InstAlias<asm # "\t$Zt, [$Rn]", 6772 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6773} 6774 6775class sve_mem_p_fill<string asm> 6776: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 6777 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6778 "", 6779 []>, Sched<[]> { 6780 bits<4> Pt; 6781 bits<5> Rn; 6782 bits<9> imm9; 6783 let Inst{31-22} = 0b1000010110; 6784 let Inst{21-16} = imm9{8-3}; 6785 let Inst{15-13} = 0b000; 6786 let Inst{12-10} = imm9{2-0}; 6787 let Inst{9-5} = Rn; 6788 let Inst{4} = 0b0; 6789 let Inst{3-0} = Pt; 6790 6791 let mayLoad = 1; 6792} 6793 6794multiclass sve_mem_p_fill<string asm> { 6795 def NAME : sve_mem_p_fill<asm>; 6796 6797 def : InstAlias<asm # "\t$Pt, [$Rn]", 6798 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6799} 6800 6801class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 6802 RegisterOperand VecList> 6803: I<(outs VecList:$Zt), iops, 6804 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 6805 "", 6806 []>, Sched<[]> { 6807 bits<3> Pg; 6808 bits<5> Rm; 6809 bits<5> Zn; 6810 bits<5> Zt; 6811 let Inst{31} = 0b1; 6812 let Inst{30} = opc{4}; 6813 let Inst{29-25} = 0b00010; 6814 let Inst{24-23} = opc{3-2}; 6815 let Inst{22-21} = 0b00; 6816 let Inst{20-16} = Rm; 6817 let Inst{15} = 0b1; 6818 let Inst{14-13} = opc{1-0}; 6819 let Inst{12-10} = Pg; 6820 let Inst{9-5} = Zn; 6821 let Inst{4-0} = Zt; 6822 6823 let mayLoad = 1; 6824} 6825 6826multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 6827 SDPatternOperator op, 6828 ValueType vt> { 6829 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 6830 asm, Z_s>; 6831 6832 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 6833 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 6834 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6835 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 6836 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6837 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 6838 6839 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 6840 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 6841} 6842 6843multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 6844 SDPatternOperator op, 6845 ValueType vt> { 6846 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 6847 asm, Z_d>; 6848 6849 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 6850 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 6851 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6852 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 6853 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6854 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 6855 6856 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 6857 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 6858} 6859 6860//===----------------------------------------------------------------------===// 6861// SVE Memory - 64-bit Gather Group 6862//===----------------------------------------------------------------------===// 6863 6864// bit xs is '1' if offsets are signed 6865// bit scaled is '1' if the offsets are scaled 6866// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 6867class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 6868 RegisterOperand zprext> 6869: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6870 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 6871 "", 6872 []>, Sched<[]> { 6873 bits<3> Pg; 6874 bits<5> Rn; 6875 bits<5> Zm; 6876 bits<5> Zt; 6877 let Inst{31-25} = 0b1100010; 6878 let Inst{24-23} = opc{3-2}; 6879 let Inst{22} = xs; 6880 let Inst{21} = scaled; 6881 let Inst{20-16} = Zm; 6882 let Inst{15} = lsl; 6883 let Inst{14-13} = opc{1-0}; 6884 let Inst{12-10} = Pg; 6885 let Inst{9-5} = Rn; 6886 let Inst{4-0} = Zt; 6887 6888 let mayLoad = 1; 6889 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6890 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6891} 6892 6893multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 6894 SDPatternOperator sxtw_op, 6895 SDPatternOperator uxtw_op, 6896 RegisterOperand sxtw_opnd, 6897 RegisterOperand uxtw_opnd, 6898 ValueType vt> { 6899 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 6900 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 6901 6902 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6903 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6904 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6905 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6906 6907 // We need a layer of indirection because early machine code passes balk at 6908 // physical register (i.e. FFR) uses that have no previous definition. 6909 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6910 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6911 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6912 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6913 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6914 } 6915 6916 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 6917 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6918 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 6919 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6920} 6921 6922multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 6923 SDPatternOperator sxtw_op, 6924 SDPatternOperator uxtw_op, 6925 RegisterOperand sxtw_opnd, 6926 RegisterOperand uxtw_opnd, 6927 ValueType vt> { 6928 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 6929 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 6930 6931 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6932 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6933 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6934 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6935 6936 // We need a layer of indirection because early machine code passes balk at 6937 // physical register (i.e. FFR) uses that have no previous definition. 6938 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6939 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6940 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6941 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6942 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6943 } 6944 6945 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 6946 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6947 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 6948 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6949} 6950 6951multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 6952 SDPatternOperator op, 6953 RegisterOperand zprext, ValueType vt> { 6954 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 6955 6956 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6957 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 6958 6959 // We need a layer of indirection because early machine code passes balk at 6960 // physical register (i.e. FFR) uses that have no previous definition. 6961 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6962 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 6963 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 6964 } 6965 6966 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 6967 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6968} 6969 6970multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 6971 SDPatternOperator op, ValueType vt> { 6972 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 6973 6974 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6975 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 6976 6977 // We need a layer of indirection because early machine code passes balk at 6978 // physical register (i.e. FFR) uses that have no previous definition. 6979 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6980 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 6981 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 6982 } 6983 6984 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 6985 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6986} 6987 6988class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 6989: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 6990 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 6991 "", 6992 []>, Sched<[]> { 6993 bits<3> Pg; 6994 bits<5> Zn; 6995 bits<5> Zt; 6996 bits<5> imm5; 6997 let Inst{31-25} = 0b1100010; 6998 let Inst{24-23} = opc{3-2}; 6999 let Inst{22-21} = 0b01; 7000 let Inst{20-16} = imm5; 7001 let Inst{15} = 0b1; 7002 let Inst{14-13} = opc{1-0}; 7003 let Inst{12-10} = Pg; 7004 let Inst{9-5} = Zn; 7005 let Inst{4-0} = Zt; 7006 7007 let mayLoad = 1; 7008 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7009 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7010} 7011 7012multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7013 SDPatternOperator op, ValueType vt> { 7014 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7015 7016 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7017 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7018 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7019 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7020 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7021 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7022 7023 // We need a layer of indirection because early machine code passes balk at 7024 // physical register (i.e. FFR) uses that have no previous definition. 7025 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7026 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7027 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7028 } 7029 7030 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7031 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7032} 7033 7034// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7035class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7036 RegisterOperand zprext> 7037: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7038 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7039 "", 7040 []>, Sched<[]> { 7041 bits<3> Pg; 7042 bits<5> Rn; 7043 bits<5> Zm; 7044 bits<4> prfop; 7045 let Inst{31-23} = 0b110001000; 7046 let Inst{22} = xs; 7047 let Inst{21} = 0b1; 7048 let Inst{20-16} = Zm; 7049 let Inst{15} = lsl; 7050 let Inst{14-13} = msz; 7051 let Inst{12-10} = Pg; 7052 let Inst{9-5} = Rn; 7053 let Inst{4} = 0b0; 7054 let Inst{3-0} = prfop; 7055 7056 let hasSideEffects = 1; 7057} 7058 7059multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7060 RegisterOperand sxtw_opnd, 7061 RegisterOperand uxtw_opnd, 7062 PatFrag op_sxtw, 7063 PatFrag op_uxtw> { 7064 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7065 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7066 7067 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7068 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7069 7070 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7071 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7072 7073} 7074 7075multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7076 RegisterOperand zprext, PatFrag frag> { 7077 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7078 7079 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7080 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7081 7082} 7083 7084class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7085: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7086 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7087 "", 7088 []>, Sched<[]> { 7089 bits<3> Pg; 7090 bits<5> Zn; 7091 bits<5> imm5; 7092 bits<4> prfop; 7093 let Inst{31-25} = 0b1100010; 7094 let Inst{24-23} = msz; 7095 let Inst{22-21} = 0b00; 7096 let Inst{20-16} = imm5; 7097 let Inst{15-13} = 0b111; 7098 let Inst{12-10} = Pg; 7099 let Inst{9-5} = Zn; 7100 let Inst{4} = 0b0; 7101 let Inst{3-0} = prfop; 7102 7103 let hasSideEffects = 1; 7104} 7105 7106multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7107 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7108 7109 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7110 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7111 7112 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7113 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7114} 7115 7116//===----------------------------------------------------------------------===// 7117// SVE Compute Vector Address Group 7118//===----------------------------------------------------------------------===// 7119 7120class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7121 ZPRRegOp zprty, RegisterOperand zprext> 7122: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7123 asm, "\t$Zd, [$Zn, $Zm]", 7124 "", 7125 []>, Sched<[]> { 7126 bits<5> Zd; 7127 bits<5> Zn; 7128 bits<5> Zm; 7129 let Inst{31-24} = 0b00000100; 7130 let Inst{23-22} = opc; 7131 let Inst{21} = 0b1; 7132 let Inst{20-16} = Zm; 7133 let Inst{15-12} = 0b1010; 7134 let Inst{11-10} = msz; 7135 let Inst{9-5} = Zn; 7136 let Inst{4-0} = Zd; 7137} 7138 7139multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7140 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7141 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7142 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7143 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7144} 7145 7146multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7147 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7148 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7149 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7150 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7151} 7152 7153multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7154 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7155 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7156 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7157 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7158} 7159 7160multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7161 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7162 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7163 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7164 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7165} 7166 7167//===----------------------------------------------------------------------===// 7168// SVE Integer Misc - Unpredicated Group 7169//===----------------------------------------------------------------------===// 7170 7171class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7172: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7173 asm, "\t$Zd, $Zn, $Zm", 7174 "", 7175 []>, Sched<[]> { 7176 bits<5> Zd; 7177 bits<5> Zm; 7178 bits<5> Zn; 7179 let Inst{31-24} = 0b00000100; 7180 let Inst{23-22} = sz; 7181 let Inst{21} = 0b1; 7182 let Inst{20-16} = Zm; 7183 let Inst{15-10} = 0b101100; 7184 let Inst{9-5} = Zn; 7185 let Inst{4-0} = Zd; 7186} 7187 7188multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7189 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7190 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7191 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7192 7193 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7194 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7195 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7196} 7197 7198class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7199: I<(outs zprty:$Zd), (ins zprty:$Zn), 7200 asm, "\t$Zd, $Zn", 7201 "", 7202 []>, Sched<[]> { 7203 bits<5> Zd; 7204 bits<5> Zn; 7205 let Inst{31-24} = 0b00000100; 7206 let Inst{23-22} = opc{7-6}; 7207 let Inst{21} = 0b1; 7208 let Inst{20-16} = opc{5-1}; 7209 let Inst{15-11} = 0b10111; 7210 let Inst{10} = opc{0}; 7211 let Inst{9-5} = Zn; 7212 let Inst{4-0} = Zd; 7213} 7214 7215multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7216 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7217 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7218 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7219 7220 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7221 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7222 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7223} 7224 7225//===----------------------------------------------------------------------===// 7226// SVE Integer Reduction Group 7227//===----------------------------------------------------------------------===// 7228 7229class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7230 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7231: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7232 asm, "\t$Vd, $Pg, $Zn", 7233 "", 7234 []>, Sched<[]> { 7235 bits<3> Pg; 7236 bits<5> Vd; 7237 bits<5> Zn; 7238 let Inst{31-24} = 0b00000100; 7239 let Inst{23-22} = sz8_32; 7240 let Inst{21} = 0b0; 7241 let Inst{20-19} = fmt; 7242 let Inst{18-16} = opc; 7243 let Inst{15-13} = 0b001; 7244 let Inst{12-10} = Pg; 7245 let Inst{9-5} = Zn; 7246 let Inst{4-0} = Vd; 7247} 7248 7249multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7250 SDPatternOperator op> { 7251 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7252 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7253 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7254 7255 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7256 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7257 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7258} 7259 7260multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7261 SDPatternOperator op> { 7262 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7263 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7264 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7265 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7266 7267 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7268 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7269 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7270 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7271} 7272 7273multiclass sve_int_reduce_1<bits<3> opc, string asm, 7274 SDPatternOperator op> { 7275 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7276 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7277 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7278 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7279 7280 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7281 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7282 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7283 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7284} 7285 7286multiclass sve_int_reduce_2<bits<3> opc, string asm, 7287 SDPatternOperator op> { 7288 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 7289 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 7290 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 7291 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 7292 7293 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7294 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7295 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7296 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7297} 7298 7299class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7300 ZPRRegOp zprty, string pg_suffix, dag iops> 7301: I<(outs zprty:$Zd), iops, 7302 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7303 "", 7304 []>, Sched<[]> { 7305 bits<3> Pg; 7306 bits<5> Zd; 7307 bits<5> Zn; 7308 let Inst{31-24} = 0b00000100; 7309 let Inst{23-22} = sz8_32; 7310 let Inst{21-19} = 0b010; 7311 let Inst{18-16} = opc; 7312 let Inst{15-13} = 0b001; 7313 let Inst{12-10} = Pg; 7314 let Inst{9-5} = Zn; 7315 let Inst{4-0} = Zd; 7316 7317 let ElementSize = zprty.ElementSize; 7318} 7319 7320multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7321let Constraints = "$Zd = $_Zd" in { 7322 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7323 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7324 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7325 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7326 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7327 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7328 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7329 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7330} 7331} 7332 7333multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7334 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7335 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7336 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7337 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7338 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7339 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7340 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7341 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7342} 7343 7344//===----------------------------------------------------------------------===// 7345// SVE Propagate Break Group 7346//===----------------------------------------------------------------------===// 7347 7348class sve_int_brkp<bits<2> opc, string asm> 7349: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7350 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7351 "", 7352 []>, Sched<[]> { 7353 bits<4> Pd; 7354 bits<4> Pg; 7355 bits<4> Pm; 7356 bits<4> Pn; 7357 let Inst{31-24} = 0b00100101; 7358 let Inst{23} = 0b0; 7359 let Inst{22} = opc{1}; 7360 let Inst{21-20} = 0b00; 7361 let Inst{19-16} = Pm; 7362 let Inst{15-14} = 0b11; 7363 let Inst{13-10} = Pg; 7364 let Inst{9} = 0b0; 7365 let Inst{8-5} = Pn; 7366 let Inst{4} = opc{0}; 7367 let Inst{3-0} = Pd; 7368 7369 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7370} 7371 7372multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7373 def NAME : sve_int_brkp<opc, asm>; 7374 7375 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7376} 7377 7378 7379//===----------------------------------------------------------------------===// 7380// SVE Partition Break Group 7381//===----------------------------------------------------------------------===// 7382 7383class sve_int_brkn<bit S, string asm> 7384: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7385 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7386 "", 7387 []>, Sched<[]> { 7388 bits<4> Pdm; 7389 bits<4> Pg; 7390 bits<4> Pn; 7391 let Inst{31-23} = 0b001001010; 7392 let Inst{22} = S; 7393 let Inst{21-14} = 0b01100001; 7394 let Inst{13-10} = Pg; 7395 let Inst{9} = 0b0; 7396 let Inst{8-5} = Pn; 7397 let Inst{4} = 0b0; 7398 let Inst{3-0} = Pdm; 7399 7400 let Constraints = "$Pdm = $_Pdm"; 7401 let Defs = !if(S, [NZCV], []); 7402} 7403 7404multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7405 def NAME : sve_int_brkn<opc, asm>; 7406 7407 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7408} 7409 7410class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 7411: I<(outs PPR8:$Pd), iops, 7412 asm, "\t$Pd, $Pg"#suffix#", $Pn", 7413 "", 7414 []>, Sched<[]> { 7415 bits<4> Pd; 7416 bits<4> Pg; 7417 bits<4> Pn; 7418 let Inst{31-24} = 0b00100101; 7419 let Inst{23-22} = opc{2-1}; 7420 let Inst{21-14} = 0b01000001; 7421 let Inst{13-10} = Pg; 7422 let Inst{9} = 0b0; 7423 let Inst{8-5} = Pn; 7424 let Inst{4} = opc{0}; 7425 let Inst{3-0} = Pd; 7426 7427 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 7428 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7429 7430} 7431 7432multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 7433 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 7434 7435 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7436} 7437 7438multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 7439 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 7440 7441 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7442} 7443 7444//===----------------------------------------------------------------------===// 7445// SVE2 String Processing Group 7446//===----------------------------------------------------------------------===// 7447 7448class sve2_char_match<bit sz, bit opc, string asm, 7449 PPRRegOp pprty, ZPRRegOp zprty> 7450: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7451 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 7452 "", 7453 []>, Sched<[]> { 7454 bits<4> Pd; 7455 bits<3> Pg; 7456 bits<5> Zm; 7457 bits<5> Zn; 7458 let Inst{31-23} = 0b010001010; 7459 let Inst{22} = sz; 7460 let Inst{21} = 0b1; 7461 let Inst{20-16} = Zm; 7462 let Inst{15-13} = 0b100; 7463 let Inst{12-10} = Pg; 7464 let Inst{9-5} = Zn; 7465 let Inst{4} = opc; 7466 let Inst{3-0} = Pd; 7467 7468 let Defs = [NZCV]; 7469} 7470 7471multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 7472 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 7473 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 7474 7475 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 7476 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7477} 7478 7479//===----------------------------------------------------------------------===// 7480// SVE2 Histogram Computation - Segment Group 7481//===----------------------------------------------------------------------===// 7482 7483class sve2_hist_gen_segment<string asm, SDPatternOperator op> 7484: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 7485 asm, "\t$Zd, $Zn, $Zm", 7486 "", 7487 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 7488 bits<5> Zd; 7489 bits<5> Zn; 7490 bits<5> Zm; 7491 let Inst{31-21} = 0b01000101001; 7492 let Inst{20-16} = Zm; 7493 let Inst{15-10} = 0b101000; 7494 let Inst{9-5} = Zn; 7495 let Inst{4-0} = Zd; 7496} 7497 7498//===----------------------------------------------------------------------===// 7499// SVE2 Histogram Computation - Vector Group 7500//===----------------------------------------------------------------------===// 7501 7502class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 7503: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7504 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 7505 "", 7506 []>, Sched<[]> { 7507 bits<5> Zd; 7508 bits<5> Zn; 7509 bits<3> Pg; 7510 bits<5> Zm; 7511 let Inst{31-23} = 0b010001011; 7512 let Inst{22} = sz; 7513 let Inst{21} = 0b1; 7514 let Inst{20-16} = Zm; 7515 let Inst{15-13} = 0b110; 7516 let Inst{12-10} = Pg; 7517 let Inst{9-5} = Zn; 7518 let Inst{4-0} = Zd; 7519} 7520 7521multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 7522 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 7523 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 7524 7525 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7526 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7527} 7528 7529//===----------------------------------------------------------------------===// 7530// SVE2 Crypto Extensions Group 7531//===----------------------------------------------------------------------===// 7532 7533class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 7534: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7535 asm, "\t$Zd, $Zn, $Zm", 7536 "", 7537 []>, Sched<[]> { 7538 bits<5> Zd; 7539 bits<5> Zn; 7540 bits<5> Zm; 7541 let Inst{31-21} = 0b01000101001; 7542 let Inst{20-16} = Zm; 7543 let Inst{15-11} = 0b11110; 7544 let Inst{10} = opc; 7545 let Inst{9-5} = Zn; 7546 let Inst{4-0} = Zd; 7547} 7548 7549multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 7550 SDPatternOperator op, ValueType vt> { 7551 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 7552 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7553} 7554 7555class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 7556: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 7557 asm, "\t$Zdn, $_Zdn, $Zm", 7558 "", 7559 []>, Sched<[]> { 7560 bits<5> Zdn; 7561 bits<5> Zm; 7562 let Inst{31-17} = 0b010001010010001; 7563 let Inst{16} = opc{1}; 7564 let Inst{15-11} = 0b11100; 7565 let Inst{10} = opc{0}; 7566 let Inst{9-5} = Zm; 7567 let Inst{4-0} = Zdn; 7568 7569 let Constraints = "$Zdn = $_Zdn"; 7570} 7571 7572multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 7573 SDPatternOperator op, ValueType vt> { 7574 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 7575 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7576} 7577 7578class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 7579: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 7580 asm, "\t$Zdn, $_Zdn", 7581 "", 7582 []>, Sched<[]> { 7583 bits<5> Zdn; 7584 let Inst{31-11} = 0b010001010010000011100; 7585 let Inst{10} = opc; 7586 let Inst{9-5} = 0b00000; 7587 let Inst{4-0} = Zdn; 7588 7589 let Constraints = "$Zdn = $_Zdn"; 7590} 7591 7592multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 7593 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 7594 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 7595} 7596 7597//===----------------------------------------------------------------------===// 7598// SVE BFloat16 Group 7599//===----------------------------------------------------------------------===// 7600 7601class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 7602: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 7603 bits<5> Zda; 7604 bits<5> Zn; 7605 let Inst{31-21} = 0b01100100011; 7606 let Inst{15-14} = opc; 7607 let Inst{13-10} = 0b0000; 7608 let Inst{9-5} = Zn; 7609 let Inst{4-0} = Zda; 7610 7611 let Constraints = "$Zda = $_Zda"; 7612 let DestructiveInstType = DestructiveOther; 7613 let ElementSize = ElementSizeH; 7614} 7615 7616class sve_bfloat_dot<string asm> 7617: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 7618 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 7619 bits<5> Zm; 7620 let Inst{20-16} = Zm; 7621} 7622 7623multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 7624 def NAME : sve_bfloat_dot<asm>; 7625 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7626} 7627 7628class sve_bfloat_dot_indexed<string asm> 7629: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 7630 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 7631 bits<2> iop; 7632 bits<3> Zm; 7633 let Inst{20-19} = iop; 7634 let Inst{18-16} = Zm; 7635} 7636 7637multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 7638 def NAME : sve_bfloat_dot_indexed<asm>; 7639 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 7640} 7641 7642class sve_bfloat_matmul<string asm> 7643: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 7644 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7645 bits<5> Zm; 7646 bits<5> Zda; 7647 bits<5> Zn; 7648 let Inst{31-21} = 0b01100100011; 7649 let Inst{20-16} = Zm; 7650 let Inst{15-10} = 0b111001; 7651 let Inst{9-5} = Zn; 7652 let Inst{4-0} = Zda; 7653 7654 let Constraints = "$Zda = $_Zda"; 7655 let DestructiveInstType = DestructiveOther; 7656 let ElementSize = ElementSizeH; 7657} 7658 7659multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 7660 def NAME : sve_bfloat_matmul<asm>; 7661 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7662} 7663 7664class sve_bfloat_matmul_longvecl<bit BT, string asm> 7665: sve_bfloat_matmul<asm> { 7666 let Inst{23} = 0b1; 7667 let Inst{14-13} = 0b00; 7668 let Inst{10} = BT; 7669} 7670 7671multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 7672 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 7673 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7674} 7675 7676class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 7677: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 7678 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 7679 bits<3> iop; 7680 bits<3> Zm; 7681 let Inst{23} = 0b1; 7682 let Inst{20-19} = iop{2-1}; 7683 let Inst{18-16} = Zm; 7684 let Inst{11} = iop{0}; 7685 let Inst{10} = BT; 7686} 7687 7688multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 7689 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 7690 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 7691} 7692 7693class sve_bfloat_convert<bit N, string asm> 7694: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 7695 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 7696 bits<5> Zd; 7697 bits<3> Pg; 7698 bits<5> Zn; 7699 let Inst{31-25} = 0b0110010; 7700 let Inst{24} = N; 7701 let Inst{23-13} = 0b10001010101; 7702 let Inst{12-10} = Pg; 7703 let Inst{9-5} = Zn; 7704 let Inst{4-0} = Zd; 7705 7706 let Constraints = "$Zd = $_Zd"; 7707 let DestructiveInstType = DestructiveOther; 7708 let hasSideEffects = 1; 7709 let ElementSize = ElementSizeS; 7710} 7711 7712multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 7713 def NAME : sve_bfloat_convert<N, asm>; 7714 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 7715} 7716 7717//===----------------------------------------------------------------------===// 7718// SVE Integer Matrix Multiply Group 7719//===----------------------------------------------------------------------===// 7720 7721class sve_int_matmul<bits<2> uns, string asm> 7722: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 7723 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7724 bits<5> Zda; 7725 bits<5> Zn; 7726 bits<5> Zm; 7727 let Inst{31-24} = 0b01000101; 7728 let Inst{23-22} = uns; 7729 let Inst{21} = 0; 7730 let Inst{20-16} = Zm; 7731 let Inst{15-10} = 0b100110; 7732 let Inst{9-5} = Zn; 7733 let Inst{4-0} = Zda; 7734 7735 let Constraints = "$Zda = $_Zda"; 7736 let DestructiveInstType = DestructiveOther; 7737 let ElementSize = ZPR32.ElementSize; 7738} 7739 7740multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 7741 def NAME : sve_int_matmul<uns, asm>; 7742 7743 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 7744} 7745 7746//===----------------------------------------------------------------------===// 7747// SVE Integer Dot Product Mixed Sign Group 7748//===----------------------------------------------------------------------===// 7749 7750class sve_int_dot_mixed<string asm> 7751: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 7752 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7753 bits<5> Zda; 7754 bits<5> Zn; 7755 bits<5> Zm; 7756 let Inst{31-21} = 0b01000100100; 7757 let Inst{20-16} = Zm; 7758 let Inst{15-10} = 0b011110; 7759 let Inst{9-5} = Zn; 7760 let Inst{4-0} = Zda; 7761 7762 let Constraints = "$Zda = $_Zda"; 7763 let DestructiveInstType = DestructiveOther; 7764 let ElementSize = ZPR32.ElementSize; 7765} 7766 7767multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 7768 def NAME : sve_int_dot_mixed<asm>; 7769 7770 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 7771} 7772 7773//===----------------------------------------------------------------------===// 7774// SVE Integer Dot Product Mixed Sign - Indexed Group 7775//===----------------------------------------------------------------------===// 7776 7777class sve_int_dot_mixed_indexed<bit U, string asm> 7778: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 7779 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 7780 bits<5> Zda; 7781 bits<5> Zn; 7782 bits<3> Zm; 7783 bits<2> idx; 7784 let Inst{31-21} = 0b01000100101; 7785 let Inst{20-19} = idx; 7786 let Inst{18-16} = Zm; 7787 let Inst{15-11} = 0b00011; 7788 let Inst{10} = U; 7789 let Inst{9-5} = Zn; 7790 let Inst{4-0} = Zda; 7791 7792 let Constraints = "$Zda = $_Zda"; 7793 let DestructiveInstType = DestructiveOther; 7794 let ElementSize = ZPR32.ElementSize; 7795} 7796 7797multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 7798 def NAME : sve_int_dot_mixed_indexed<U, asm>; 7799 7800 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 7801} 7802 7803//===----------------------------------------------------------------------===// 7804// SVE Floating Point Matrix Multiply Accumulate Group 7805//===----------------------------------------------------------------------===// 7806 7807class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 7808: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 7809 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7810 bits<5> Zda; 7811 bits<5> Zn; 7812 bits<5> Zm; 7813 let Inst{31-23} = 0b011001001; 7814 let Inst{22} = sz; 7815 let Inst{21} = 1; 7816 let Inst{20-16} = Zm; 7817 let Inst{15-10} = 0b111001; 7818 let Inst{9-5} = Zn; 7819 let Inst{4-0} = Zda; 7820 7821 let Constraints = "$Zda = $_Zda"; 7822 let DestructiveInstType = DestructiveOther; 7823 let ElementSize = zprty.ElementSize; 7824} 7825 7826multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 7827 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 7828 7829 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 7830} 7831 7832//===----------------------------------------------------------------------===// 7833// SVE Memory - Contiguous Load And Replicate 256-bit Group 7834//===----------------------------------------------------------------------===// 7835 7836class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 7837: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 7838 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 7839 bits<5> Zt; 7840 bits<5> Rn; 7841 bits<3> Pg; 7842 bits<4> imm4; 7843 let Inst{31-25} = 0b1010010; 7844 let Inst{24-23} = sz; 7845 let Inst{22-20} = 0b010; 7846 let Inst{19-16} = imm4; 7847 let Inst{15-13} = 0b001; 7848 let Inst{12-10} = Pg; 7849 let Inst{9-5} = Rn; 7850 let Inst{4-0} = Zt; 7851 7852 let mayLoad = 1; 7853} 7854 7855multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 7856 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 7857 def NAME : sve_mem_ldor_si<sz, asm, listty>; 7858 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7859 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7860 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7861 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 7862 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 7863 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 7864 7865 // Base addressing mode 7866 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 7867 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 7868 let AddedComplexity = 2 in { 7869 // Reg + Imm addressing mode 7870 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 7871 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 7872 } 7873} 7874 7875class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 7876 RegisterOperand gprty> 7877: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7878 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 7879 bits<5> Zt; 7880 bits<3> Pg; 7881 bits<5> Rn; 7882 bits<5> Rm; 7883 let Inst{31-25} = 0b1010010; 7884 let Inst{24-23} = sz; 7885 let Inst{22-21} = 0b01; 7886 let Inst{20-16} = Rm; 7887 let Inst{15-13} = 0; 7888 let Inst{12-10} = Pg; 7889 let Inst{9-5} = Rn; 7890 let Inst{4-0} = Zt; 7891 7892 let mayLoad = 1; 7893} 7894 7895multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 7896 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 7897 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 7898 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 7899 7900 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7901 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7902 7903 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 7904 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 7905} 7906 7907//===----------------------------------------------------------------------===// 7908// SVE Interleave 128-bit Elements Group 7909//===----------------------------------------------------------------------===// 7910 7911class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 7912: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 7913 asm, "\t$Zd, $Zn, $Zm", 7914 "", 7915 []>, Sched<[]> { 7916 bits<5> Zd; 7917 bits<5> Zm; 7918 bits<5> Zn; 7919 let Inst{31-21} = 0b00000101101; 7920 let Inst{20-16} = Zm; 7921 let Inst{15-13} = 0b000; 7922 let Inst{12-11} = opc; 7923 let Inst{10} = P; 7924 let Inst{9-5} = Zn; 7925 let Inst{4-0} = Zd; 7926} 7927 7928multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 7929 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 7930 7931 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 7932 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 7933 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 7934 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 7935 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 7936 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 7937 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 7938 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 7939} 7940 7941/// Addressing modes 7942def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 7943def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 7944 7945def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>; 7946def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>; 7947def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>; 7948def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>; 7949 7950// Predicated pseudo floating point two operand instructions. 7951multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 7952 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 7953 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 7954 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 7955 7956 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 7957 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 7958 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 7959 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7960 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7961 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 7962} 7963 7964// Predicated pseudo integer two operand instructions. 7965multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 7966 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 7967 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 7968 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 7969 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 7970 7971 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 7972 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 7973 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7974 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 7975} 7976 7977// As sve_int_bin_pred but when only i32 and i64 vector types are required. 7978multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 7979 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 7980 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 7981 7982 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 7983 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 7984} 7985 7986// Predicated pseudo integer two operand instructions. Second operand is an 7987// immediate specified by imm_[bhsd]. 7988multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 7989 ComplexPattern imm_b, ComplexPattern imm_h, 7990 ComplexPattern imm_s, ComplexPattern imm_d> { 7991 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 7992 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 7993 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 7994 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 7995 7996 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 7997 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 7998 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 7999 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8000} 8001