1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEPrefetchOperand : AsmOperandClass { 38 let Name = "SVEPrefetch"; 39 let ParserMethod = "tryParsePrefetch<true>"; 40 let PredicateMethod = "isPrefetch"; 41 let RenderMethod = "addPrefetchOperands"; 42} 43 44def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 45 return (((uint32_t)Imm) <= 15); 46 }]> { 47 let PrintMethod = "printPrefetchOp<true>"; 48 let ParserMatchClass = SVEPrefetchOperand; 49} 50 51class SVELogicalImmOperand<int Width> : AsmOperandClass { 52 let Name = "SVELogicalImm" # Width; 53 let DiagnosticType = "LogicalSecondSource"; 54 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 55 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 56} 57 58def sve_logical_imm8 : Operand<i64> { 59 let ParserMatchClass = SVELogicalImmOperand<8>; 60 let PrintMethod = "printLogicalImm<int8_t>"; 61 62 let MCOperandPredicate = [{ 63 if (!MCOp.isImm()) 64 return false; 65 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 66 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 67 }]; 68} 69 70def sve_logical_imm16 : Operand<i64> { 71 let ParserMatchClass = SVELogicalImmOperand<16>; 72 let PrintMethod = "printLogicalImm<int16_t>"; 73 74 let MCOperandPredicate = [{ 75 if (!MCOp.isImm()) 76 return false; 77 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 78 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 79 }]; 80} 81 82def sve_logical_imm32 : Operand<i64> { 83 let ParserMatchClass = SVELogicalImmOperand<32>; 84 let PrintMethod = "printLogicalImm<int32_t>"; 85 86 let MCOperandPredicate = [{ 87 if (!MCOp.isImm()) 88 return false; 89 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 90 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 91 }]; 92} 93 94class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 95 let Name = "SVEPreferredLogicalImm" # Width; 96 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 97 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 98} 99 100def sve_preferred_logical_imm16 : Operand<i64> { 101 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 102 let PrintMethod = "printSVELogicalImm<int16_t>"; 103 104 let MCOperandPredicate = [{ 105 if (!MCOp.isImm()) 106 return false; 107 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 108 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 109 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 110 }]; 111} 112 113def sve_preferred_logical_imm32 : Operand<i64> { 114 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 115 let PrintMethod = "printSVELogicalImm<int32_t>"; 116 117 let MCOperandPredicate = [{ 118 if (!MCOp.isImm()) 119 return false; 120 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 121 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 122 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 123 }]; 124} 125 126def sve_preferred_logical_imm64 : Operand<i64> { 127 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 128 let PrintMethod = "printSVELogicalImm<int64_t>"; 129 130 let MCOperandPredicate = [{ 131 if (!MCOp.isImm()) 132 return false; 133 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 134 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 135 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 136 }]; 137} 138 139class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 140 let Name = "SVELogicalImm" # Width # "Not"; 141 let DiagnosticType = "LogicalSecondSource"; 142 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 143 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 144} 145 146def sve_logical_imm8_not : Operand<i64> { 147 let ParserMatchClass = SVELogicalImmNotOperand<8>; 148} 149 150def sve_logical_imm16_not : Operand<i64> { 151 let ParserMatchClass = SVELogicalImmNotOperand<16>; 152} 153 154def sve_logical_imm32_not : Operand<i64> { 155 let ParserMatchClass = SVELogicalImmNotOperand<32>; 156} 157 158class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 159 : AsmOperandClass { 160 let Name = "SVE" # Infix # "Imm" # ElementWidth; 161 let DiagnosticType = "Invalid" # Name; 162 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 163 let ParserMethod = "tryParseImmWithOptionalShift"; 164 let PredicateMethod = Predicate; 165} 166 167def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 168def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 169def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 170def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 171 172def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 173def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 174def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 175def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 176 177class imm8_opt_lsl<int ElementWidth, string printType, 178 AsmOperandClass OpndClass> 179 : Operand<i32> { 180 let EncoderMethod = "getImm8OptLsl"; 181 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 182 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 183 let ParserMatchClass = OpndClass; 184 let MIOperandInfo = (ops i32imm, i32imm); 185} 186 187def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 188def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 189def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 190def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 191 192def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 193def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 194def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 195def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 196 197def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 198def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 199def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 200def SVEAddSubImm64Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 201 202def SVELogicalImm8Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8>", []>; 203def SVELogicalImm16Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16>", []>; 204def SVELogicalImm32Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32>", []>; 205def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 206 207def SVELogicalImm8NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; 208def SVELogicalImm16NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; 209def SVELogicalImm32NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; 210def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; 211 212def SVE8BitLslImm : ComplexPattern<i32, 2, "SelectSVE8BitLslImm", [imm]>; 213 214def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 215def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 216def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 217def SVEArithUImm64Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i64>", []>; 218def SVEArithSImmPat : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 219 220def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 221def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 222def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 223def SVEShiftImmL64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 63>", []>; 224def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 225def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 226def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 227def SVEShiftImmR64 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 64, true>", []>; 228 229def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; 230 231class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 232 let Name = "SVEExactFPImmOperand" # Suffix; 233 let DiagnosticType = "Invalid" # Name; 234 let ParserMethod = "tryParseFPImm<false>"; 235 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 236 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 237} 238 239class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 240 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 241 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 242} 243 244def sve_fpimm_half_one 245 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 246 "AArch64ExactFPImm::one">; 247def sve_fpimm_half_two 248 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 249 "AArch64ExactFPImm::two">; 250def sve_fpimm_zero_one 251 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 252 "AArch64ExactFPImm::one">; 253 254def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 255 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 256}]> { 257 let ParserMatchClass = Imm1_16Operand; 258 let EncoderMethod = "getSVEIncDecImm"; 259 let DecoderMethod = "DecodeSVEIncDecImm"; 260} 261 262// This allows i32 immediate extraction from i64 based arithmetic. 263def sve_cnt_mul_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 264def sve_cnt_shl_imm : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, true>">; 265 266 267def sve_ext_imm_0_1 : ComplexPattern<i32, 1, "SelectEXTImm<1, 8>">; 268def sve_ext_imm_0_3 : ComplexPattern<i32, 1, "SelectEXTImm<3, 4>">; 269def sve_ext_imm_0_7 : ComplexPattern<i32, 1, "SelectEXTImm<7, 2>">; 270def sve_ext_imm_0_15 : ComplexPattern<i32, 1, "SelectEXTImm<15, 1>">; 271 272def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), 273 (int_aarch64_sve_cntp node:$pred, node:$src2), [{ 274 return N->hasOneUse(); 275}]>; 276 277//===----------------------------------------------------------------------===// 278// SVE PTrue - These are used extensively throughout the pattern matching so 279// it's important we define them first. 280//===----------------------------------------------------------------------===// 281 282class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 283 ValueType vt, SDPatternOperator op> 284: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 285 asm, "\t$Pd, $pattern", 286 "", 287 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 288 bits<4> Pd; 289 bits<5> pattern; 290 let Inst{31-24} = 0b00100101; 291 let Inst{23-22} = sz8_64; 292 let Inst{21-19} = 0b011; 293 let Inst{18-17} = opc{2-1}; 294 let Inst{16} = opc{0}; 295 let Inst{15-10} = 0b111000; 296 let Inst{9-5} = pattern; 297 let Inst{4} = 0b0; 298 let Inst{3-0} = Pd; 299 300 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 301 let ElementSize = pprty.ElementSize; 302 let isReMaterializable = 1; 303} 304 305multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 306 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 307 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 308 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 309 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 310 311 def : InstAlias<asm # "\t$Pd", 312 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 313 def : InstAlias<asm # "\t$Pd", 314 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 315 def : InstAlias<asm # "\t$Pd", 316 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 317 def : InstAlias<asm # "\t$Pd", 318 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 319} 320 321def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 322def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 323 324let Predicates = [HasSVE] in { 325 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 326 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 327} 328 329//===----------------------------------------------------------------------===// 330// SVE pattern match helpers. 331//===----------------------------------------------------------------------===// 332 333class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 334 Instruction inst> 335: Pat<(vtd (op vt1:$Op1)), 336 (inst $Op1)>; 337 338class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 339 ValueType vts, Instruction inst> 340: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 341 (inst $Op3, $Op1, $Op2)>; 342 343 344multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 345 ValueType vts, Instruction inst> { 346 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))), 347 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 348 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)), 349 (inst $Op3, $Op1, $Op2)>; 350} 351 352// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 353// type of rounding. This is matched by timm0_1 in pattern below and ignored. 354class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 355 ValueType vts, Instruction inst> 356: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 357 (inst $Op3, $Op1, $Op2)>; 358 359class SVE_1_Op_Imm_OptLsl_Reverse_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 360 ValueType it, ComplexPattern cpx, Instruction inst> 361 : Pat<(vt (op (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))), (vt zprty:$Op1))), 362 (inst $Op1, i32:$imm, i32:$shift)>; 363 364class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 365 ValueType it, ComplexPattern cpx, Instruction inst> 366 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm, i32:$shift)))))), 367 (inst $Op1, i32:$imm, i32:$shift)>; 368 369class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op, 370 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 371 : Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (AArch64dup (it (cpx i32:$imm)))))), 372 (inst $Op1, i32:$imm)>; 373 374class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 375 ValueType it, ComplexPattern cpx, Instruction inst> 376 : Pat<(vt (op (vt zprty:$Op1), (vt (AArch64dup (it (cpx i64:$imm)))))), 377 (inst $Op1, i64:$imm)>; 378 379class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 380 ValueType vt2, Instruction inst> 381: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 382 (inst $Op1, $Op2)>; 383 384class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 385 ValueType pt, ValueType vt1, ValueType vt2, 386 Instruction inst> 387: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)), 388 (inst $Op1, $Op2)>; 389 390class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op, 391 ValueType pt, ValueType vt1, ValueType vt2, 392 Instruction inst> 393: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)), 394 (inst $Op1, $Op2, $Op3)>; 395 396class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 397 ValueType vt2, ValueType vt3, Instruction inst> 398: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 399 (inst $Op1, $Op2, $Op3)>; 400 401multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 402 ValueType vt2, ValueType vt3, Instruction inst> { 403 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)), 404 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 405 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)), 406 (inst $Op1, $Op2, $Op3)>; 407} 408 409class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 410 ValueType vt2, ValueType vt3, ValueType vt4, 411 Instruction inst> 412: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 413 (inst $Op1, $Op2, $Op3, $Op4)>; 414 415class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 416 ValueType vt2, Operand ImmTy, Instruction inst> 417: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 418 (inst $Op1, ImmTy:$Op2)>; 419 420class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 421 ValueType vt2, ValueType vt3, Operand ImmTy, 422 Instruction inst> 423: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 424 (inst $Op1, $Op2, ImmTy:$Op3)>; 425 426class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 427 ValueType vt2, ValueType vt3, ValueType vt4, 428 Operand ImmTy, Instruction inst> 429: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 430 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 431 432def SVEDup0 : ComplexPattern<i64, 0, "SelectDupZero", []>; 433def SVEDup0Undef : ComplexPattern<i64, 0, "SelectDupZeroOrUndef", []>; 434 435let AddedComplexity = 1 in { 436class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 437 ValueType vt2, ValueType vt3, Instruction inst> 438: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 439 (inst $Op1, $Op2, $Op3)>; 440 441class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 442 ValueType vt1, ValueType vt2, 443 Operand vt3, Instruction inst> 444: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 445 (inst $Op1, $Op2, vt3:$Op3)>; 446} 447 448// 449// Common but less generic patterns. 450// 451 452class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 453 Instruction inst, Instruction ptrue> 454: Pat<(vtd (op vt1:$Op1)), 455 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 456 457class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 458 ValueType vt2, Instruction inst, Instruction ptrue> 459: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 460 (inst (ptrue 31), $Op1, $Op2)>; 461 462class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 463 ValueType inreg_vt, Instruction inst> 464: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 465 (inst $PassThru, $Pg, $Src)>; 466 467multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt, 468 ValueType inreg_vt, Instruction inst> { 469 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))), 470 (inst (IMPLICIT_DEF), $Pg, $Src)>; 471 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)), 472 (inst $PassThru, $Pg, $Src)>; 473} 474 475class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 476 ValueType pt, ValueType it, 477 ComplexPattern cast, Instruction inst> 478: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))), 479 (inst $Pg, $Rn, i32:$imm)>; 480 481class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 482 ValueType pt, ValueType it, 483 ComplexPattern cast, Instruction inst> 484: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (AArch64dup (it (cast i32:$imm)))))), 485 (inst $Rn, i32:$imm)>; 486 487// 488// Pseudo -> Instruction mappings 489// 490def getSVEPseudoMap : InstrMapping { 491 let FilterClass = "SVEPseudo2Instr"; 492 let RowFields = ["PseudoName"]; 493 let ColFields = ["IsInstr"]; 494 let KeyCol = ["0"]; 495 let ValueCols = [["1"]]; 496} 497 498class SVEPseudo2Instr<string name, bit instr> { 499 string PseudoName = name; 500 bit IsInstr = instr; 501} 502 503// Lookup e.g. DIV -> DIVR 504def getSVERevInstr : InstrMapping { 505 let FilterClass = "SVEInstr2Rev"; 506 let RowFields = ["InstrName"]; 507 let ColFields = ["isReverseInstr"]; 508 let KeyCol = ["0"]; 509 let ValueCols = [["1"]]; 510} 511 512// Lookup e.g. DIVR -> DIV 513def getSVENonRevInstr : InstrMapping { 514 let FilterClass = "SVEInstr2Rev"; 515 let RowFields = ["InstrName"]; 516 let ColFields = ["isReverseInstr"]; 517 let KeyCol = ["1"]; 518 let ValueCols = [["0"]]; 519} 520 521class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 522 string InstrName = !if(name1IsReverseInstr, name1, name2); 523 bit isReverseInstr = name1IsReverseInstr; 524} 525 526// 527// Pseudos for destructive operands 528// 529let hasNoSchedulingInfo = 1 in { 530 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 531 FalseLanesEnum flags = FalseLanesNone> 532 : SVEPseudo2Instr<name, 0>, 533 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 534 let FalseLanes = flags; 535 } 536 537 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 538 FalseLanesEnum flags = FalseLanesNone> 539 : SVEPseudo2Instr<name, 0>, 540 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 541 let FalseLanes = flags; 542 } 543 544 class PredThreeOpPseudo<string name, ZPRRegOp zprty, 545 FalseLanesEnum flags = FalseLanesNone> 546 : SVEPseudo2Instr<name, 0>, 547 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> { 548 let FalseLanes = flags; 549 } 550} 551 552// 553// Pseudos for passthru operands 554// 555let hasNoSchedulingInfo = 1 in { 556 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty> 557 : SVEPseudo2Instr<name, 0>, 558 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>; 559} 560 561//===----------------------------------------------------------------------===// 562// SVE Predicate Misc Group 563//===----------------------------------------------------------------------===// 564 565class sve_int_pfalse<bits<6> opc, string asm> 566: I<(outs PPR8:$Pd), (ins), 567 asm, "\t$Pd", 568 "", 569 []>, Sched<[]> { 570 bits<4> Pd; 571 let Inst{31-24} = 0b00100101; 572 let Inst{23-22} = opc{5-4}; 573 let Inst{21-19} = 0b011; 574 let Inst{18-16} = opc{3-1}; 575 let Inst{15-10} = 0b111001; 576 let Inst{9} = opc{0}; 577 let Inst{8-4} = 0b00000; 578 let Inst{3-0} = Pd; 579 580 let isReMaterializable = 1; 581} 582 583class sve_int_ptest<bits<6> opc, string asm> 584: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 585 asm, "\t$Pg, $Pn", 586 "", 587 []>, Sched<[]> { 588 bits<4> Pg; 589 bits<4> Pn; 590 let Inst{31-24} = 0b00100101; 591 let Inst{23-22} = opc{5-4}; 592 let Inst{21-19} = 0b010; 593 let Inst{18-16} = opc{3-1}; 594 let Inst{15-14} = 0b11; 595 let Inst{13-10} = Pg; 596 let Inst{9} = opc{0}; 597 let Inst{8-5} = Pn; 598 let Inst{4-0} = 0b00000; 599 600 let Defs = [NZCV]; 601 let isCompare = 1; 602} 603 604class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 605 PPRRegOp pprty> 606: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 607 asm, "\t$Pdn, $Pg, $_Pdn", 608 "", 609 []>, Sched<[]> { 610 bits<4> Pdn; 611 bits<4> Pg; 612 let Inst{31-24} = 0b00100101; 613 let Inst{23-22} = sz8_64; 614 let Inst{21-19} = 0b011; 615 let Inst{18-16} = opc{4-2}; 616 let Inst{15-11} = 0b11000; 617 let Inst{10-9} = opc{1-0}; 618 let Inst{8-5} = Pg; 619 let Inst{4} = 0; 620 let Inst{3-0} = Pdn; 621 622 let Constraints = "$Pdn = $_Pdn"; 623 let Defs = [NZCV]; 624} 625 626multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 627 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 628 629 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 630} 631 632multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 633 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 634 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 635 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 636 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 637 638 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 639 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 640 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 641 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 642} 643 644//===----------------------------------------------------------------------===// 645// SVE Predicate Count Group 646//===----------------------------------------------------------------------===// 647 648class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 649 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 650: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 651 asm, "\t$Rdn, $Pg", 652 "", 653 []>, Sched<[]> { 654 bits<5> Rdn; 655 bits<4> Pg; 656 let Inst{31-24} = 0b00100101; 657 let Inst{23-22} = sz8_64; 658 let Inst{21-19} = 0b101; 659 let Inst{18-16} = opc{4-2}; 660 let Inst{15-11} = 0b10001; 661 let Inst{10-9} = opc{1-0}; 662 let Inst{8-5} = Pg; 663 let Inst{4-0} = Rdn; 664 665 // Signed 32bit forms require their GPR operand printed. 666 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 667 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 668 !strconcat(asm, "\t$Rdn, $Pg")); 669 let Constraints = "$Rdn = $_Rdn"; 670} 671 672multiclass sve_int_count_r_s32<bits<5> opc, string asm, 673 SDPatternOperator op> { 674 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 675 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 676 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 677 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 678 679 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 680 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 681 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 682 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 683 684 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 685 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 686 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 687 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 688 689 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 690 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 691 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 692 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 693 694 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 695 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 696 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 697 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 698} 699 700multiclass sve_int_count_r_u32<bits<5> opc, string asm, 701 SDPatternOperator op> { 702 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 703 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 704 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 705 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 706 707 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 708 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 709 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 710 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 711 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 712 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 713 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 714 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 715} 716 717multiclass sve_int_count_r_x64<bits<5> opc, string asm, 718 SDPatternOperator op, 719 SDPatternOperator combine_op = null_frag> { 720 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 721 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 722 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 723 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 724 725 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 726 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 727 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 728 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 729 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 730 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 731 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 732 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 733 734 // Combine cntp with combine_op 735 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))), 736 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 737 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))), 738 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 739 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))), 740 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 741 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))), 742 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 743} 744 745class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 746 ZPRRegOp zprty, PPRRegOp pprty> 747: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 748 asm, "\t$Zdn, $Pm", 749 "", 750 []>, Sched<[]> { 751 bits<4> Pm; 752 bits<5> Zdn; 753 let Inst{31-24} = 0b00100101; 754 let Inst{23-22} = sz8_64; 755 let Inst{21-19} = 0b101; 756 let Inst{18-16} = opc{4-2}; 757 let Inst{15-11} = 0b10000; 758 let Inst{10-9} = opc{1-0}; 759 let Inst{8-5} = Pm; 760 let Inst{4-0} = Zdn; 761 762 let Constraints = "$Zdn = $_Zdn"; 763 let DestructiveInstType = DestructiveOther; 764 let ElementSize = ElementSizeNone; 765} 766 767multiclass sve_int_count_v<bits<5> opc, string asm, 768 SDPatternOperator op = null_frag> { 769 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 770 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 771 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 772 773 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 774 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 775 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 776 777 def : InstAlias<asm # "\t$Zdn, $Pm", 778 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 779 def : InstAlias<asm # "\t$Zdn, $Pm", 780 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 781 def : InstAlias<asm # "\t$Zdn, $Pm", 782 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 783} 784 785class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 786 PPRRegOp pprty> 787: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 788 asm, "\t$Rd, $Pg, $Pn", 789 "", 790 []>, Sched<[]> { 791 bits<4> Pg; 792 bits<4> Pn; 793 bits<5> Rd; 794 let Inst{31-24} = 0b00100101; 795 let Inst{23-22} = sz8_64; 796 let Inst{21-19} = 0b100; 797 let Inst{18-16} = opc{3-1}; 798 let Inst{15-14} = 0b10; 799 let Inst{13-10} = Pg; 800 let Inst{9} = opc{0}; 801 let Inst{8-5} = Pn; 802 let Inst{4-0} = Rd; 803} 804 805multiclass sve_int_pcount_pred<bits<4> opc, string asm, 806 SDPatternOperator int_op> { 807 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 808 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 809 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 810 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 811 812 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 813 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 814 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 815 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 816} 817 818//===----------------------------------------------------------------------===// 819// SVE Element Count Group 820//===----------------------------------------------------------------------===// 821 822class sve_int_count<bits<3> opc, string asm> 823: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 824 asm, "\t$Rd, $pattern, mul $imm4", 825 "", 826 []>, Sched<[]> { 827 bits<5> Rd; 828 bits<4> imm4; 829 bits<5> pattern; 830 let Inst{31-24} = 0b00000100; 831 let Inst{23-22} = opc{2-1}; 832 let Inst{21-20} = 0b10; 833 let Inst{19-16} = imm4; 834 let Inst{15-11} = 0b11100; 835 let Inst{10} = opc{0}; 836 let Inst{9-5} = pattern; 837 let Inst{4-0} = Rd; 838} 839 840multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 841 def NAME : sve_int_count<opc, asm>; 842 843 def : InstAlias<asm # "\t$Rd, $pattern", 844 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 845 def : InstAlias<asm # "\t$Rd", 846 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 847 848 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm i32:$imm))), 849 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 850 851 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (i64 (sve_cnt_shl_imm i32:$imm)))), 852 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 853 854 def : Pat<(i64 (op sve_pred_enum:$pattern)), 855 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 856} 857 858class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 859: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 860 asm, "\t$Zdn, $pattern, mul $imm4", 861 "", 862 []>, Sched<[]> { 863 bits<5> Zdn; 864 bits<5> pattern; 865 bits<4> imm4; 866 let Inst{31-24} = 0b00000100; 867 let Inst{23-22} = opc{4-3}; 868 let Inst{21} = 0b1; 869 let Inst{20} = opc{2}; 870 let Inst{19-16} = imm4; 871 let Inst{15-12} = 0b1100; 872 let Inst{11-10} = opc{1-0}; 873 let Inst{9-5} = pattern; 874 let Inst{4-0} = Zdn; 875 876 let Constraints = "$Zdn = $_Zdn"; 877 let DestructiveInstType = DestructiveOther; 878 let ElementSize = ElementSizeNone; 879} 880 881multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 882 SDPatternOperator op = null_frag, 883 ValueType vt = OtherVT> { 884 def NAME : sve_int_countvlv<opc, asm, zprty>; 885 886 def : InstAlias<asm # "\t$Zdn, $pattern", 887 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 888 def : InstAlias<asm # "\t$Zdn", 889 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 890 891 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 892 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 893} 894 895class sve_int_pred_pattern_a<bits<3> opc, string asm> 896: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 897 asm, "\t$Rdn, $pattern, mul $imm4", 898 "", 899 []>, Sched<[]> { 900 bits<5> Rdn; 901 bits<5> pattern; 902 bits<4> imm4; 903 let Inst{31-24} = 0b00000100; 904 let Inst{23-22} = opc{2-1}; 905 let Inst{21-20} = 0b11; 906 let Inst{19-16} = imm4; 907 let Inst{15-11} = 0b11100; 908 let Inst{10} = opc{0}; 909 let Inst{9-5} = pattern; 910 let Inst{4-0} = Rdn; 911 912 let Constraints = "$Rdn = $_Rdn"; 913} 914 915multiclass sve_int_pred_pattern_a<bits<3> opc, string asm> { 916 def NAME : sve_int_pred_pattern_a<opc, asm>; 917 918 def : InstAlias<asm # "\t$Rdn, $pattern", 919 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 920 def : InstAlias<asm # "\t$Rdn", 921 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 922} 923 924class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 925 RegisterOperand st> 926: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 927 asm, "\t$Rdn, $pattern, mul $imm4", 928 "", 929 []>, Sched<[]> { 930 bits<5> Rdn; 931 bits<5> pattern; 932 bits<4> imm4; 933 let Inst{31-24} = 0b00000100; 934 let Inst{23-22} = opc{4-3}; 935 let Inst{21} = 0b1; 936 let Inst{20} = opc{2}; 937 let Inst{19-16} = imm4; 938 let Inst{15-12} = 0b1111; 939 let Inst{11-10} = opc{1-0}; 940 let Inst{9-5} = pattern; 941 let Inst{4-0} = Rdn; 942 943 // Signed 32bit forms require their GPR operand printed. 944 let AsmString = !if(!eq(opc{2,0}, 0b00), 945 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 946 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 947 948 let Constraints = "$Rdn = $_Rdn"; 949} 950 951multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 952 SDPatternOperator op> { 953 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 954 955 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 956 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 957 def : InstAlias<asm # "\t$Rd, $Rn", 958 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 959 960 // NOTE: Register allocation doesn't like tied operands of differing register 961 // class, hence the extra INSERT_SUBREG complication. 962 963 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 964 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 965 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 966 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 967} 968 969multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 970 SDPatternOperator op> { 971 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 972 973 def : InstAlias<asm # "\t$Rdn, $pattern", 974 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 975 def : InstAlias<asm # "\t$Rdn", 976 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 977 978 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 979 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 980} 981 982multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 983 SDPatternOperator op> { 984 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 985 986 def : InstAlias<asm # "\t$Rdn, $pattern", 987 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 988 def : InstAlias<asm # "\t$Rdn", 989 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 990 991 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 992 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 993} 994 995 996//===----------------------------------------------------------------------===// 997// SVE Permute - Cross Lane Group 998//===----------------------------------------------------------------------===// 999 1000class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1001 ValueType vt, RegisterClass srcRegType, 1002 SDPatternOperator op> 1003: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 1004 asm, "\t$Zd, $Rn", 1005 "", 1006 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 1007 bits<5> Rn; 1008 bits<5> Zd; 1009 let Inst{31-24} = 0b00000101; 1010 let Inst{23-22} = sz8_64; 1011 let Inst{21-10} = 0b100000001110; 1012 let Inst{9-5} = Rn; 1013 let Inst{4-0} = Zd; 1014} 1015 1016multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 1017 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 1018 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 1019 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 1020 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 1021 1022 def : InstAlias<"mov $Zd, $Rn", 1023 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 1024 def : InstAlias<"mov $Zd, $Rn", 1025 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 1026 def : InstAlias<"mov $Zd, $Rn", 1027 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 1028 def : InstAlias<"mov $Zd, $Rn", 1029 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 1030} 1031 1032class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 1033 ZPRRegOp zprty> 1034: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 1035 asm, "\t$Zd, $Zn$idx", 1036 "", 1037 []>, Sched<[]> { 1038 bits<5> Zd; 1039 bits<5> Zn; 1040 bits<7> idx; 1041 let Inst{31-24} = 0b00000101; 1042 let Inst{23-22} = {?,?}; // imm3h 1043 let Inst{21} = 0b1; 1044 let Inst{20-16} = tsz; 1045 let Inst{15-10} = 0b001000; 1046 let Inst{9-5} = Zn; 1047 let Inst{4-0} = Zd; 1048} 1049 1050multiclass sve_int_perm_dup_i<string asm> { 1051 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 1052 let Inst{23-22} = idx{5-4}; 1053 let Inst{20-17} = idx{3-0}; 1054 } 1055 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 1056 let Inst{23-22} = idx{4-3}; 1057 let Inst{20-18} = idx{2-0}; 1058 } 1059 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 1060 let Inst{23-22} = idx{3-2}; 1061 let Inst{20-19} = idx{1-0}; 1062 } 1063 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 1064 let Inst{23-22} = idx{2-1}; 1065 let Inst{20} = idx{0}; 1066 } 1067 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 1068 let Inst{23-22} = idx{1-0}; 1069 } 1070 1071 def : InstAlias<"mov $Zd, $Zn$idx", 1072 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 1073 def : InstAlias<"mov $Zd, $Zn$idx", 1074 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 1075 def : InstAlias<"mov $Zd, $Zn$idx", 1076 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1077 def : InstAlias<"mov $Zd, $Zn$idx", 1078 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1079 def : InstAlias<"mov $Zd, $Zn$idx", 1080 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1081 def : InstAlias<"mov $Zd, $Bn", 1082 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1083 def : InstAlias<"mov $Zd, $Hn", 1084 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1085 def : InstAlias<"mov $Zd, $Sn", 1086 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1087 def : InstAlias<"mov $Zd, $Dn", 1088 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1089 def : InstAlias<"mov $Zd, $Qn", 1090 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1091 1092 // Duplicate extracted element of vector into all vector elements 1093 def : Pat<(nxv16i8 (AArch64dup (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))), 1094 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>; 1095 def : Pat<(nxv8i16 (AArch64dup (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1096 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1097 def : Pat<(nxv4i32 (AArch64dup (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1098 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1099 def : Pat<(nxv2i64 (AArch64dup (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1100 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1101 def : Pat<(nxv8f16 (AArch64dup (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1102 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1103 def : Pat<(nxv8bf16 (AArch64dup (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1104 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1105 def : Pat<(nxv4f16 (AArch64dup (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1106 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1107 def : Pat<(nxv2f16 (AArch64dup (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1108 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1109 def : Pat<(nxv4f32 (AArch64dup (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1110 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1111 def : Pat<(nxv2f32 (AArch64dup (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1112 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1113 def : Pat<(nxv2f64 (AArch64dup (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1114 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1115} 1116 1117class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1118 RegisterOperand VecList> 1119: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1120 asm, "\t$Zd, $Zn, $Zm", 1121 "", 1122 []>, Sched<[]> { 1123 bits<5> Zd; 1124 bits<5> Zm; 1125 bits<5> Zn; 1126 let Inst{31-24} = 0b00000101; 1127 let Inst{23-22} = sz8_64; 1128 let Inst{21} = 0b1; 1129 let Inst{20-16} = Zm; 1130 let Inst{15-13} = 0b001; 1131 let Inst{12-11} = opc; 1132 let Inst{10} = 0b0; 1133 let Inst{9-5} = Zn; 1134 let Inst{4-0} = Zd; 1135} 1136 1137multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1138 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1139 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1140 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1141 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1142 1143 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1144 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1145 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1146 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1147 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1148 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1149 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1150 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1151 1152 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1153 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1154 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1155 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1156 1157 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1158 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1159 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1160 1161 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1162} 1163 1164multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1165 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1166 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1167 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1168 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1169 1170 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1171 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1172 nxv16i8:$Op2, zsub1), 1173 nxv16i8:$Op3))>; 1174 1175 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1176 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1177 nxv8i16:$Op2, zsub1), 1178 nxv8i16:$Op3))>; 1179 1180 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1181 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1182 nxv4i32:$Op2, zsub1), 1183 nxv4i32:$Op3))>; 1184 1185 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1186 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1187 nxv2i64:$Op2, zsub1), 1188 nxv2i64:$Op3))>; 1189 1190 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1191 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1192 nxv8f16:$Op2, zsub1), 1193 nxv8i16:$Op3))>; 1194 1195 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1196 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1197 nxv4f32:$Op2, zsub1), 1198 nxv4i32:$Op3))>; 1199 1200 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1201 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1202 nxv2f64:$Op2, zsub1), 1203 nxv2i64:$Op3))>; 1204 1205 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1206 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1207 nxv8bf16:$Op2, zsub1), 1208 nxv8i16:$Op3))>; 1209} 1210 1211class sve2_int_perm_tbx<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1212: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1213 asm, "\t$Zd, $Zn, $Zm", 1214 "", 1215 []>, Sched<[]> { 1216 bits<5> Zd; 1217 bits<5> Zm; 1218 bits<5> Zn; 1219 let Inst{31-24} = 0b00000101; 1220 let Inst{23-22} = sz8_64; 1221 let Inst{21} = 0b1; 1222 let Inst{20-16} = Zm; 1223 let Inst{15-10} = 0b001011; 1224 let Inst{9-5} = Zn; 1225 let Inst{4-0} = Zd; 1226 1227 let Constraints = "$Zd = $_Zd"; 1228} 1229 1230multiclass sve2_int_perm_tbx<string asm, SDPatternOperator op> { 1231 def _B : sve2_int_perm_tbx<0b00, asm, ZPR8>; 1232 def _H : sve2_int_perm_tbx<0b01, asm, ZPR16>; 1233 def _S : sve2_int_perm_tbx<0b10, asm, ZPR32>; 1234 def _D : sve2_int_perm_tbx<0b11, asm, ZPR64>; 1235 1236 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1237 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1238 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1239 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1240 1241 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1242 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1243 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1244 1245 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1246} 1247 1248class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1249: I<(outs zprty:$Zd), (ins zprty:$Zn), 1250 asm, "\t$Zd, $Zn", 1251 "", 1252 []>, Sched<[]> { 1253 bits<5> Zd; 1254 bits<5> Zn; 1255 let Inst{31-24} = 0b00000101; 1256 let Inst{23-22} = sz8_64; 1257 let Inst{21-10} = 0b111000001110; 1258 let Inst{9-5} = Zn; 1259 let Inst{4-0} = Zd; 1260} 1261 1262multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1263 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1264 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1265 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1266 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1267 1268 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1269 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1270 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1271 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1272 1273 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1274 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1275 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1276 1277 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1278} 1279 1280class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty> 1281: I<(outs pprty:$Pd), (ins pprty:$Pn), 1282 asm, "\t$Pd, $Pn", 1283 "", 1284 []>, Sched<[]> { 1285 bits<4> Pd; 1286 bits<4> Pn; 1287 let Inst{31-24} = 0b00000101; 1288 let Inst{23-22} = sz8_64; 1289 let Inst{21-9} = 0b1101000100000; 1290 let Inst{8-5} = Pn; 1291 let Inst{4} = 0b0; 1292 let Inst{3-0} = Pd; 1293} 1294 1295multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator op> { 1296 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8>; 1297 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16>; 1298 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32>; 1299 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64>; 1300 1301 def : SVE_1_Op_Pat<nxv16i1, op, nxv16i1, !cast<Instruction>(NAME # _B)>; 1302 def : SVE_1_Op_Pat<nxv8i1, op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1303 def : SVE_1_Op_Pat<nxv4i1, op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1304 def : SVE_1_Op_Pat<nxv2i1, op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1305} 1306 1307class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1308 ZPRRegOp zprty1, ZPRRegOp zprty2> 1309: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1310 asm, "\t$Zd, $Zn", 1311 "", []>, Sched<[]> { 1312 bits<5> Zd; 1313 bits<5> Zn; 1314 let Inst{31-24} = 0b00000101; 1315 let Inst{23-22} = sz16_64; 1316 let Inst{21-18} = 0b1100; 1317 let Inst{17-16} = opc; 1318 let Inst{15-10} = 0b001110; 1319 let Inst{9-5} = Zn; 1320 let Inst{4-0} = Zd; 1321} 1322 1323multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1324 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1325 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1326 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1327 1328 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1329 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1330 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1331} 1332 1333class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1334 RegisterClass srcRegType> 1335: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1336 asm, "\t$Zdn, $Rm", 1337 "", 1338 []>, Sched<[]> { 1339 bits<5> Rm; 1340 bits<5> Zdn; 1341 let Inst{31-24} = 0b00000101; 1342 let Inst{23-22} = sz8_64; 1343 let Inst{21-10} = 0b100100001110; 1344 let Inst{9-5} = Rm; 1345 let Inst{4-0} = Zdn; 1346 1347 let Constraints = "$Zdn = $_Zdn"; 1348 let DestructiveInstType = DestructiveOther; 1349} 1350 1351multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1352 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1353 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1354 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1355 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1356 1357 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1358 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1359 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1360 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1361} 1362 1363class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1364 FPRasZPROperand srcOpType> 1365: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm), 1366 asm, "\t$Zdn, $Vm", 1367 "", 1368 []>, Sched<[]> { 1369 bits<5> Vm; 1370 bits<5> Zdn; 1371 let Inst{31-24} = 0b00000101; 1372 let Inst{23-22} = sz8_64; 1373 let Inst{21-10} = 0b110100001110; 1374 let Inst{9-5} = Vm; 1375 let Inst{4-0} = Zdn; 1376 1377 let Constraints = "$Zdn = $_Zdn"; 1378 let DestructiveInstType = DestructiveOther; 1379} 1380 1381multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1382 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>; 1383 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>; 1384 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>; 1385 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>; 1386 1387 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)), 1388 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1389 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)), 1390 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>; 1391 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)), 1392 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>; 1393 1394 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)), 1395 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1396 1397 // Keep integer insertions within the vector unit. 1398 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))), 1399 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>; 1400 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))), 1401 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>; 1402 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))), 1403 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>; 1404 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))), 1405 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>; 1406 1407} 1408 1409//===----------------------------------------------------------------------===// 1410// SVE Permute - Extract Group 1411//===----------------------------------------------------------------------===// 1412 1413class sve_int_perm_extract_i<string asm> 1414: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1415 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1416 "", []>, Sched<[]> { 1417 bits<5> Zdn; 1418 bits<5> Zm; 1419 bits<8> imm8; 1420 let Inst{31-21} = 0b00000101001; 1421 let Inst{20-16} = imm8{7-3}; 1422 let Inst{15-13} = 0b000; 1423 let Inst{12-10} = imm8{2-0}; 1424 let Inst{9-5} = Zm; 1425 let Inst{4-0} = Zdn; 1426 1427 let Constraints = "$Zdn = $_Zdn"; 1428 let DestructiveInstType = DestructiveOther; 1429 let ElementSize = ElementSizeNone; 1430} 1431 1432multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1433 def NAME : sve_int_perm_extract_i<asm>; 1434 1435 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1436 !cast<Instruction>(NAME)>; 1437} 1438 1439class sve2_int_perm_extract_i_cons<string asm> 1440: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1441 asm, "\t$Zd, $Zn, $imm8", 1442 "", []>, Sched<[]> { 1443 bits<5> Zd; 1444 bits<5> Zn; 1445 bits<8> imm8; 1446 let Inst{31-21} = 0b00000101011; 1447 let Inst{20-16} = imm8{7-3}; 1448 let Inst{15-13} = 0b000; 1449 let Inst{12-10} = imm8{2-0}; 1450 let Inst{9-5} = Zn; 1451 let Inst{4-0} = Zd; 1452} 1453 1454//===----------------------------------------------------------------------===// 1455// SVE Vector Select Group 1456//===----------------------------------------------------------------------===// 1457 1458class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1459: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1460 asm, "\t$Zd, $Pg, $Zn, $Zm", 1461 "", 1462 []>, Sched<[]> { 1463 bits<4> Pg; 1464 bits<5> Zd; 1465 bits<5> Zm; 1466 bits<5> Zn; 1467 let Inst{31-24} = 0b00000101; 1468 let Inst{23-22} = sz8_64; 1469 let Inst{21} = 0b1; 1470 let Inst{20-16} = Zm; 1471 let Inst{15-14} = 0b11; 1472 let Inst{13-10} = Pg; 1473 let Inst{9-5} = Zn; 1474 let Inst{4-0} = Zd; 1475} 1476 1477multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1478 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1479 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1480 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1481 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1482 1483 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1484 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1485 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1486 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1487 1488 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1489 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 1490 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1491 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 1492 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1493 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1494 1495 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1496 1497 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1498 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1499 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1500 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1501 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1502 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1503 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1504 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1505} 1506 1507 1508//===----------------------------------------------------------------------===// 1509// SVE Predicate Logical Operations Group 1510//===----------------------------------------------------------------------===// 1511 1512class sve_int_pred_log<bits<4> opc, string asm> 1513: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1514 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1515 "", 1516 []>, Sched<[]> { 1517 bits<4> Pd; 1518 bits<4> Pg; 1519 bits<4> Pm; 1520 bits<4> Pn; 1521 let Inst{31-24} = 0b00100101; 1522 let Inst{23-22} = opc{3-2}; 1523 let Inst{21-20} = 0b00; 1524 let Inst{19-16} = Pm; 1525 let Inst{15-14} = 0b01; 1526 let Inst{13-10} = Pg; 1527 let Inst{9} = opc{1}; 1528 let Inst{8-5} = Pn; 1529 let Inst{4} = opc{0}; 1530 let Inst{3-0} = Pd; 1531 1532 // SEL has no predication qualifier. 1533 let AsmString = !if(!eq(opc, 0b0011), 1534 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1535 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1536 1537 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1538 1539} 1540 1541multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1542 SDPatternOperator op_nopred = null_frag> { 1543 def NAME : sve_int_pred_log<opc, asm>; 1544 1545 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1546 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1547 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1548 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1549 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1550 !cast<Instruction>(NAME), PTRUE_B>; 1551 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1552 !cast<Instruction>(NAME), PTRUE_H>; 1553 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1554 !cast<Instruction>(NAME), PTRUE_S>; 1555 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1556 !cast<Instruction>(NAME), PTRUE_D>; 1557} 1558 1559//===----------------------------------------------------------------------===// 1560// SVE Logical Mask Immediate Group 1561//===----------------------------------------------------------------------===// 1562 1563class sve_int_log_imm<bits<2> opc, string asm> 1564: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1565 asm, "\t$Zdn, $_Zdn, $imms13", 1566 "", []>, Sched<[]> { 1567 bits<5> Zdn; 1568 bits<13> imms13; 1569 let Inst{31-24} = 0b00000101; 1570 let Inst{23-22} = opc; 1571 let Inst{21-18} = 0b0000; 1572 let Inst{17-5} = imms13; 1573 let Inst{4-0} = Zdn; 1574 1575 let Constraints = "$Zdn = $_Zdn"; 1576 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1577 let DestructiveInstType = DestructiveOther; 1578 let ElementSize = ElementSizeNone; 1579} 1580 1581multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1582 def NAME : sve_int_log_imm<opc, asm>; 1583 1584 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1585 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1586 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1587 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1588 1589 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1590 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1591 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1592 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1593 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1594 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1595 1596 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1597 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1598 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1599 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1600 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1601 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1602 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1603 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1604} 1605 1606multiclass sve_int_log_imm_bic<SDPatternOperator op> { 1607 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>; 1608 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>; 1609 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>; 1610 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>; 1611} 1612 1613class sve_int_dup_mask_imm<string asm> 1614: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1615 asm, "\t$Zd, $imms", 1616 "", 1617 []>, Sched<[]> { 1618 bits<5> Zd; 1619 bits<13> imms; 1620 let Inst{31-18} = 0b00000101110000; 1621 let Inst{17-5} = imms; 1622 let Inst{4-0} = Zd; 1623 1624 let isReMaterializable = 1; 1625 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1626} 1627 1628multiclass sve_int_dup_mask_imm<string asm> { 1629 def NAME : sve_int_dup_mask_imm<asm>; 1630 1631 def : InstAlias<"dupm $Zd, $imm", 1632 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1633 def : InstAlias<"dupm $Zd, $imm", 1634 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1635 def : InstAlias<"dupm $Zd, $imm", 1636 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1637 1638 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1639 def : InstAlias<"mov $Zd, $imm", 1640 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1641 def : InstAlias<"mov $Zd, $imm", 1642 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1643 def : InstAlias<"mov $Zd, $imm", 1644 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1645} 1646 1647//===----------------------------------------------------------------------===// 1648// SVE Integer Arithmetic - Unpredicated Group. 1649//===----------------------------------------------------------------------===// 1650 1651class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1652 ZPRRegOp zprty> 1653: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1654 asm, "\t$Zd, $Zn, $Zm", 1655 "", []>, Sched<[]> { 1656 bits<5> Zd; 1657 bits<5> Zm; 1658 bits<5> Zn; 1659 let Inst{31-24} = 0b00000100; 1660 let Inst{23-22} = sz8_64; 1661 let Inst{21} = 0b1; 1662 let Inst{20-16} = Zm; 1663 let Inst{15-13} = 0b000; 1664 let Inst{12-10} = opc; 1665 let Inst{9-5} = Zn; 1666 let Inst{4-0} = Zd; 1667} 1668 1669multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> { 1670 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1671 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1672 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1673 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1674 1675 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1676 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1677 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1678 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1679} 1680 1681//===----------------------------------------------------------------------===// 1682// SVE Floating Point Arithmetic - Predicated Group 1683//===----------------------------------------------------------------------===// 1684 1685class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1686 ZPRRegOp zprty, 1687 Operand imm_ty> 1688: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1689 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1690 "", 1691 []>, Sched<[]> { 1692 bits<3> Pg; 1693 bits<5> Zdn; 1694 bit i1; 1695 let Inst{31-24} = 0b01100101; 1696 let Inst{23-22} = sz; 1697 let Inst{21-19} = 0b011; 1698 let Inst{18-16} = opc; 1699 let Inst{15-13} = 0b100; 1700 let Inst{12-10} = Pg; 1701 let Inst{9-6} = 0b0000; 1702 let Inst{5} = i1; 1703 let Inst{4-0} = Zdn; 1704 1705 let Constraints = "$Zdn = $_Zdn"; 1706 let DestructiveInstType = DestructiveOther; 1707 let ElementSize = zprty.ElementSize; 1708} 1709 1710multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, Operand imm_ty> { 1711 def _H : sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1712 def _S : sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1713 def _D : sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1714} 1715 1716class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1717 ZPRRegOp zprty> 1718: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1719 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1720 "", 1721 []>, Sched<[]> { 1722 bits<3> Pg; 1723 bits<5> Zdn; 1724 bits<5> Zm; 1725 let Inst{31-24} = 0b01100101; 1726 let Inst{23-22} = sz; 1727 let Inst{21-20} = 0b00; 1728 let Inst{19-16} = opc; 1729 let Inst{15-13} = 0b100; 1730 let Inst{12-10} = Pg; 1731 let Inst{9-5} = Zm; 1732 let Inst{4-0} = Zdn; 1733 1734 let Constraints = "$Zdn = $_Zdn"; 1735 let DestructiveInstType = DestructiveOther; 1736 let ElementSize = zprty.ElementSize; 1737} 1738 1739multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1740 SDPatternOperator op, DestructiveInstTypeEnum flags, 1741 string revname="", bit isReverseInstr=0> { 1742 let DestructiveInstType = flags in { 1743 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1744 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1745 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1746 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1747 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1748 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1749 } 1750 1751 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1752 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1753 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1754} 1755 1756multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1757 SDPatternOperator op> { 1758 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1759 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1760 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1761 1762 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1763 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1764 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1765} 1766 1767multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1768 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1769 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1770 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1771 1772 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1773 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1774 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1775} 1776 1777class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1778: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, imm32_0_7:$imm3), 1779 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1780 "", 1781 []>, Sched<[]> { 1782 bits<5> Zdn; 1783 bits<5> Zm; 1784 bits<3> imm3; 1785 let Inst{31-24} = 0b01100101; 1786 let Inst{23-22} = sz; 1787 let Inst{21-19} = 0b010; 1788 let Inst{18-16} = imm3; 1789 let Inst{15-10} = 0b100000; 1790 let Inst{9-5} = Zm; 1791 let Inst{4-0} = Zdn; 1792 1793 let Constraints = "$Zdn = $_Zdn"; 1794 let DestructiveInstType = DestructiveOther; 1795 let ElementSize = ElementSizeNone; 1796} 1797 1798multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 1799 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 1800 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 1801 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 1802 1803 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 imm32_0_7:$imm))), 1804 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, imm32_0_7:$imm)>; 1805 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 imm32_0_7:$imm))), 1806 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, imm32_0_7:$imm)>; 1807 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 imm32_0_7:$imm))), 1808 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, imm32_0_7:$imm)>; 1809} 1810 1811//===----------------------------------------------------------------------===// 1812// SVE Floating Point Arithmetic - Unpredicated Group 1813//===----------------------------------------------------------------------===// 1814 1815class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 1816: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1817 asm, "\t$Zd, $Zn, $Zm", 1818 "", 1819 []>, Sched<[]> { 1820 bits<5> Zd; 1821 bits<5> Zm; 1822 bits<5> Zn; 1823 let Inst{31-24} = 0b01100101; 1824 let Inst{23-22} = sz; 1825 let Inst{21} = 0b0; 1826 let Inst{20-16} = Zm; 1827 let Inst{15-13} = 0b000; 1828 let Inst{12-10} = opc; 1829 let Inst{9-5} = Zn; 1830 let Inst{4-0} = Zd; 1831} 1832 1833multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 1834 SDPatternOperator predicated_op = null_frag> { 1835 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1836 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1837 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1838 1839 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1840 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1841 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1842 1843 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1844 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1845 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1846} 1847 1848multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 1849 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 1850 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 1851 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 1852 1853 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1854 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1855 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1856} 1857 1858//===----------------------------------------------------------------------===// 1859// SVE Floating Point Fused Multiply-Add Group 1860//===----------------------------------------------------------------------===// 1861 1862class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 1863: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 1864 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 1865 "", 1866 []>, Sched<[]> { 1867 bits<3> Pg; 1868 bits<5> Zda; 1869 bits<5> Zm; 1870 bits<5> Zn; 1871 let Inst{31-24} = 0b01100101; 1872 let Inst{23-22} = sz; 1873 let Inst{21} = 0b1; 1874 let Inst{20-16} = Zm; 1875 let Inst{15} = 0b0; 1876 let Inst{14-13} = opc; 1877 let Inst{12-10} = Pg; 1878 let Inst{9-5} = Zn; 1879 let Inst{4-0} = Zda; 1880 1881 let Constraints = "$Zda = $_Zda"; 1882 let ElementSize = zprty.ElementSize; 1883} 1884 1885multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps, 1886 SDPatternOperator op, string revname, 1887 bit isReverseInstr=0> { 1888 let DestructiveInstType = DestructiveTernaryCommWithRev in { 1889 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>, 1890 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1891 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>, 1892 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1893 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>, 1894 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1895 } 1896 1897 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1898 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1899 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1900} 1901 1902class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 1903 ZPRRegOp zprty> 1904: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 1905 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 1906 "", 1907 []>, Sched<[]> { 1908 bits<3> Pg; 1909 bits<5> Za; 1910 bits<5> Zdn; 1911 bits<5> Zm; 1912 let Inst{31-24} = 0b01100101; 1913 let Inst{23-22} = sz; 1914 let Inst{21} = 0b1; 1915 let Inst{20-16} = Za; 1916 let Inst{15} = 0b1; 1917 let Inst{14-13} = opc; 1918 let Inst{12-10} = Pg; 1919 let Inst{9-5} = Zm; 1920 let Inst{4-0} = Zdn; 1921 1922 let Constraints = "$Zdn = $_Zdn"; 1923 let DestructiveInstType = DestructiveOther; 1924 let ElementSize = zprty.ElementSize; 1925} 1926 1927multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op, 1928 string revname, bit isReverseInstr> { 1929 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>, 1930 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1931 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>, 1932 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1933 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>, 1934 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1935 1936 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1937 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1938 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1939} 1940 1941multiclass sve_fp_3op_p_zds_zx<SDPatternOperator op, SDPatternOperator rev_op> { 1942 def _UNDEF_H : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 1943 def _UNDEF_S : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 1944 def _UNDEF_D : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 1945} 1946 1947//===----------------------------------------------------------------------===// 1948// SVE Floating Point Multiply-Add - Indexed Group 1949//===----------------------------------------------------------------------===// 1950 1951class sve_fp_fma_by_indexed_elem<bits<2> sz, bit opc, string asm, 1952 ZPRRegOp zprty1, 1953 ZPRRegOp zprty2, Operand itype> 1954: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 1955 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 1956 bits<5> Zda; 1957 bits<5> Zn; 1958 let Inst{31-24} = 0b01100100; 1959 let Inst{23-22} = sz; 1960 let Inst{21} = 0b1; 1961 let Inst{15-11} = 0; 1962 let Inst{10} = opc; 1963 let Inst{9-5} = Zn; 1964 let Inst{4-0} = Zda; 1965 1966 let Constraints = "$Zda = $_Zda"; 1967 let DestructiveInstType = DestructiveOther; 1968 let ElementSize = ElementSizeNone; 1969} 1970 1971multiclass sve_fp_fma_by_indexed_elem<bit opc, string asm, 1972 SDPatternOperator op> { 1973 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 1974 bits<3> Zm; 1975 bits<3> iop; 1976 let Inst{22} = iop{2}; 1977 let Inst{20-19} = iop{1-0}; 1978 let Inst{18-16} = Zm; 1979 } 1980 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 1981 bits<3> Zm; 1982 bits<2> iop; 1983 let Inst{20-19} = iop; 1984 let Inst{18-16} = Zm; 1985 } 1986 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 1987 bits<4> Zm; 1988 bit iop; 1989 let Inst{20} = iop; 1990 let Inst{19-16} = Zm; 1991 } 1992 1993 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 1994 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 1995 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 1996 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 1997 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 1998 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 1999} 2000 2001 2002//===----------------------------------------------------------------------===// 2003// SVE Floating Point Multiply - Indexed Group 2004//===----------------------------------------------------------------------===// 2005 2006class sve_fp_fmul_by_indexed_elem<bits<2> sz, string asm, ZPRRegOp zprty, 2007 ZPRRegOp zprty2, Operand itype> 2008: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 2009 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2010 bits<5> Zd; 2011 bits<5> Zn; 2012 let Inst{31-24} = 0b01100100; 2013 let Inst{23-22} = sz; 2014 let Inst{21} = 0b1; 2015 let Inst{15-10} = 0b001000; 2016 let Inst{9-5} = Zn; 2017 let Inst{4-0} = Zd; 2018} 2019 2020multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 2021 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2022 bits<3> Zm; 2023 bits<3> iop; 2024 let Inst{22} = iop{2}; 2025 let Inst{20-19} = iop{1-0}; 2026 let Inst{18-16} = Zm; 2027 } 2028 def _S : sve_fp_fmul_by_indexed_elem<0b10, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2029 bits<3> Zm; 2030 bits<2> iop; 2031 let Inst{20-19} = iop; 2032 let Inst{18-16} = Zm; 2033 } 2034 def _D : sve_fp_fmul_by_indexed_elem<0b11, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2035 bits<4> Zm; 2036 bit iop; 2037 let Inst{20} = iop; 2038 let Inst{19-16} = Zm; 2039 } 2040 2041 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 2042 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 2043 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 2044 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 2045 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 2046 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 2047} 2048 2049//===----------------------------------------------------------------------===// 2050// SVE Floating Point Complex Multiply-Add Group 2051//===----------------------------------------------------------------------===// 2052 2053class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 2054: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 2055 complexrotateop:$imm), 2056 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 2057 "", []>, Sched<[]> { 2058 bits<5> Zda; 2059 bits<3> Pg; 2060 bits<5> Zn; 2061 bits<5> Zm; 2062 bits<2> imm; 2063 let Inst{31-24} = 0b01100100; 2064 let Inst{23-22} = sz; 2065 let Inst{21} = 0; 2066 let Inst{20-16} = Zm; 2067 let Inst{15} = 0; 2068 let Inst{14-13} = imm; 2069 let Inst{12-10} = Pg; 2070 let Inst{9-5} = Zn; 2071 let Inst{4-0} = Zda; 2072 2073 let Constraints = "$Zda = $_Zda"; 2074 let DestructiveInstType = DestructiveOther; 2075 let ElementSize = zprty.ElementSize; 2076} 2077 2078multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 2079 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 2080 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 2081 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 2082 2083 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 2084 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2085 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 2086 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2087 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 2088 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2089} 2090 2091//===----------------------------------------------------------------------===// 2092// SVE Floating Point Complex Multiply-Add - Indexed Group 2093//===----------------------------------------------------------------------===// 2094 2095class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 2096 ZPRRegOp zprty, 2097 ZPRRegOp zprty2, Operand itype> 2098: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 2099 complexrotateop:$imm), 2100 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 2101 "", []>, Sched<[]> { 2102 bits<5> Zda; 2103 bits<5> Zn; 2104 bits<2> imm; 2105 let Inst{31-24} = 0b01100100; 2106 let Inst{23-22} = sz; 2107 let Inst{21} = 0b1; 2108 let Inst{15-12} = 0b0001; 2109 let Inst{11-10} = imm; 2110 let Inst{9-5} = Zn; 2111 let Inst{4-0} = Zda; 2112 2113 let Constraints = "$Zda = $_Zda"; 2114 let DestructiveInstType = DestructiveOther; 2115 let ElementSize = ElementSizeNone; 2116} 2117 2118multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 2119 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 2120 bits<3> Zm; 2121 bits<2> iop; 2122 let Inst{20-19} = iop; 2123 let Inst{18-16} = Zm; 2124 } 2125 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 2126 bits<4> Zm; 2127 bits<1> iop; 2128 let Inst{20} = iop; 2129 let Inst{19-16} = Zm; 2130 } 2131 2132 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2133 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2134 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2135 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2136} 2137 2138//===----------------------------------------------------------------------===// 2139// SVE Floating Point Complex Addition Group 2140//===----------------------------------------------------------------------===// 2141 2142class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2143: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2144 complexrotateopodd:$imm), 2145 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2146 "", 2147 []>, Sched<[]> { 2148 bits<5> Zdn; 2149 bits<5> Zm; 2150 bits<3> Pg; 2151 bit imm; 2152 let Inst{31-24} = 0b01100100; 2153 let Inst{23-22} = sz; 2154 let Inst{21-17} = 0; 2155 let Inst{16} = imm; 2156 let Inst{15-13} = 0b100; 2157 let Inst{12-10} = Pg; 2158 let Inst{9-5} = Zm; 2159 let Inst{4-0} = Zdn; 2160 2161 let Constraints = "$Zdn = $_Zdn"; 2162 let DestructiveInstType = DestructiveOther; 2163 let ElementSize = zprty.ElementSize; 2164} 2165 2166multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2167 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2168 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2169 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2170 2171 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2172 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2173 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2174 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2175 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2176 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2177} 2178 2179//===----------------------------------------------------------------------===// 2180// SVE2 Floating Point Convert Group 2181//===----------------------------------------------------------------------===// 2182 2183class sve2_fp_convert_precision<bits<4> opc, string asm, 2184 ZPRRegOp zprty1, ZPRRegOp zprty2> 2185: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2186 asm, "\t$Zd, $Pg/m, $Zn", 2187 "", 2188 []>, Sched<[]> { 2189 bits<5> Zd; 2190 bits<5> Zn; 2191 bits<3> Pg; 2192 let Inst{31-24} = 0b01100100; 2193 let Inst{23-22} = opc{3-2}; 2194 let Inst{21-18} = 0b0010; 2195 let Inst{17-16} = opc{1-0}; 2196 let Inst{15-13} = 0b101; 2197 let Inst{12-10} = Pg; 2198 let Inst{9-5} = Zn; 2199 let Inst{4-0} = Zd; 2200 2201 let Constraints = "$Zd = $_Zd"; 2202} 2203 2204multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2205 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2206 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2207 2208 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2209 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2210} 2211 2212multiclass sve2_fp_convert_up_long<string asm, string op> { 2213 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2214 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2215 2216 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2217 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2218} 2219 2220multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2221 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2222 2223 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2224} 2225 2226//===----------------------------------------------------------------------===// 2227// SVE2 Floating Point Pairwise Group 2228//===----------------------------------------------------------------------===// 2229 2230class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2231 ZPRRegOp zprty> 2232: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2233 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2234 "", 2235 []>, Sched<[]> { 2236 bits<3> Pg; 2237 bits<5> Zm; 2238 bits<5> Zdn; 2239 let Inst{31-24} = 0b01100100; 2240 let Inst{23-22} = sz; 2241 let Inst{21-19} = 0b010; 2242 let Inst{18-16} = opc; 2243 let Inst{15-13} = 0b100; 2244 let Inst{12-10} = Pg; 2245 let Inst{9-5} = Zm; 2246 let Inst{4-0} = Zdn; 2247 2248 let Constraints = "$Zdn = $_Zdn"; 2249 let DestructiveInstType = DestructiveOther; 2250 let ElementSize = zprty.ElementSize; 2251} 2252 2253multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2254 SDPatternOperator op> { 2255 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2256 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2257 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2258 2259 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2260 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2261 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2262} 2263 2264//===----------------------------------------------------------------------===// 2265// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2266//===----------------------------------------------------------------------===// 2267 2268class sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm> 2269: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2270 VectorIndexH32b:$iop), 2271 asm, "\t$Zda, $Zn, $Zm$iop", 2272 "", 2273 []>, Sched<[]> { 2274 bits<5> Zda; 2275 bits<5> Zn; 2276 bits<3> Zm; 2277 bits<3> iop; 2278 let Inst{31-21} = 0b01100100101; 2279 let Inst{20-19} = iop{2-1}; 2280 let Inst{18-16} = Zm; 2281 let Inst{15-14} = 0b01; 2282 let Inst{13} = opc{1}; 2283 let Inst{12} = 0b0; 2284 let Inst{11} = iop{0}; 2285 let Inst{10} = opc{0}; 2286 let Inst{9-5} = Zn; 2287 let Inst{4-0} = Zda; 2288 2289 let Constraints = "$Zda = $_Zda"; 2290 let DestructiveInstType = DestructiveOther; 2291 let ElementSize = ElementSizeNone; 2292} 2293 2294multiclass sve2_fp_mla_long_by_indexed_elem<bits<2> opc, string asm, 2295 SDPatternOperator op> { 2296 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2297 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2298} 2299 2300//===----------------------------------------------------------------------===// 2301// SVE2 Floating Point Widening Multiply-Add Group 2302//===----------------------------------------------------------------------===// 2303 2304class sve2_fp_mla_long<bits<2> opc, string asm> 2305: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2306 asm, "\t$Zda, $Zn, $Zm", 2307 "", 2308 []>, Sched<[]> { 2309 bits<5> Zda; 2310 bits<5> Zn; 2311 bits<5> Zm; 2312 let Inst{31-21} = 0b01100100101; 2313 let Inst{20-16} = Zm; 2314 let Inst{15-14} = 0b10; 2315 let Inst{13} = opc{1}; 2316 let Inst{12-11} = 0b00; 2317 let Inst{10} = opc{0}; 2318 let Inst{9-5} = Zn; 2319 let Inst{4-0} = Zda; 2320 2321 let Constraints = "$Zda = $_Zda"; 2322 let DestructiveInstType = DestructiveOther; 2323 let ElementSize = ElementSizeNone; 2324} 2325 2326multiclass sve2_fp_mla_long<bits<2> opc, string asm, SDPatternOperator op> { 2327 def NAME : sve2_fp_mla_long<opc, asm>; 2328 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 2329} 2330 2331//===----------------------------------------------------------------------===// 2332// SVE Stack Allocation Group 2333//===----------------------------------------------------------------------===// 2334 2335class sve_int_arith_vl<bit opc, string asm> 2336: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2337 asm, "\t$Rd, $Rn, $imm6", 2338 "", 2339 []>, Sched<[]> { 2340 bits<5> Rd; 2341 bits<5> Rn; 2342 bits<6> imm6; 2343 let Inst{31-23} = 0b000001000; 2344 let Inst{22} = opc; 2345 let Inst{21} = 0b1; 2346 let Inst{20-16} = Rn; 2347 let Inst{15-11} = 0b01010; 2348 let Inst{10-5} = imm6; 2349 let Inst{4-0} = Rd; 2350} 2351 2352class sve_int_read_vl_a<bit op, bits<5> opc2, string asm> 2353: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2354 asm, "\t$Rd, $imm6", 2355 "", 2356 []>, Sched<[]> { 2357 bits<5> Rd; 2358 bits<6> imm6; 2359 let Inst{31-23} = 0b000001001; 2360 let Inst{22} = op; 2361 let Inst{21} = 0b1; 2362 let Inst{20-16} = opc2{4-0}; 2363 let Inst{15-11} = 0b01010; 2364 let Inst{10-5} = imm6; 2365 let Inst{4-0} = Rd; 2366} 2367 2368//===----------------------------------------------------------------------===// 2369// SVE Permute - In Lane Group 2370//===----------------------------------------------------------------------===// 2371 2372class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2373 ZPRRegOp zprty> 2374: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2375 asm, "\t$Zd, $Zn, $Zm", 2376 "", 2377 []>, Sched<[]> { 2378 bits<5> Zd; 2379 bits<5> Zm; 2380 bits<5> Zn; 2381 let Inst{31-24} = 0b00000101; 2382 let Inst{23-22} = sz8_64; 2383 let Inst{21} = 0b1; 2384 let Inst{20-16} = Zm; 2385 let Inst{15-13} = 0b011; 2386 let Inst{12-10} = opc; 2387 let Inst{9-5} = Zn; 2388 let Inst{4-0} = Zd; 2389} 2390 2391multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2392 SDPatternOperator op> { 2393 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2394 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2395 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2396 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2397 2398 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2399 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2400 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2401 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2402 2403 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2404 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2405 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2406 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2407 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2408 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2409 2410 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2411} 2412 2413//===----------------------------------------------------------------------===// 2414// SVE Floating Point Unary Operations Group 2415//===----------------------------------------------------------------------===// 2416 2417class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2418 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2419: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2420 asm, "\t$Zd, $Pg/m, $Zn", 2421 "", 2422 []>, Sched<[]> { 2423 bits<3> Pg; 2424 bits<5> Zd; 2425 bits<5> Zn; 2426 let Inst{31-24} = 0b01100101; 2427 let Inst{23-22} = opc{6-5}; 2428 let Inst{21} = 0b0; 2429 let Inst{20-16} = opc{4-0}; 2430 let Inst{15-13} = 0b101; 2431 let Inst{12-10} = Pg; 2432 let Inst{9-5} = Zn; 2433 let Inst{4-0} = Zd; 2434 2435 let Constraints = "$Zd = $_Zd"; 2436 let DestructiveInstType = DestructiveOther; 2437 let ElementSize = Sz; 2438} 2439 2440multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2441 RegisterOperand i_zprtype, 2442 RegisterOperand o_zprtype, 2443 SDPatternOperator int_op, 2444 SDPatternOperator ir_op, ValueType vt1, 2445 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2446 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2447 2448 // convert vt1 to a packed type for the intrinsic patterns 2449 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2450 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2451 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2452 1 : vt1); 2453 2454 // convert vt3 to a packed type for the intrinsic patterns 2455 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2456 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2457 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2458 1 : vt3); 2459 2460 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2461 2462 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2463} 2464 2465multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2466 RegisterOperand i_zprtype, 2467 RegisterOperand o_zprtype, 2468 SDPatternOperator int_op, 2469 SDPatternOperator ir_op, ValueType vt1, 2470 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2471 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>; 2472 2473 // convert vt1 to a packed type for the intrinsic patterns 2474 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2475 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2476 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2477 1 : vt1); 2478 2479 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2480 2481 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2482} 2483 2484multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2485 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>; 2486 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>; 2487 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>; 2488 2489 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2490 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2491 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2492 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2493 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2494 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2495} 2496 2497multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2498 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2499 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2500 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2501 2502 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2503 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2504 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2505} 2506 2507multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2508 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2509 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2510} 2511 2512//===----------------------------------------------------------------------===// 2513// SVE Floating Point Unary Operations - Unpredicated Group 2514//===----------------------------------------------------------------------===// 2515 2516class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2517 ZPRRegOp zprty> 2518: I<(outs zprty:$Zd), (ins zprty:$Zn), 2519 asm, "\t$Zd, $Zn", 2520 "", 2521 []>, Sched<[]> { 2522 bits<5> Zd; 2523 bits<5> Zn; 2524 let Inst{31-24} = 0b01100101; 2525 let Inst{23-22} = sz; 2526 let Inst{21-19} = 0b001; 2527 let Inst{18-16} = opc; 2528 let Inst{15-10} = 0b001100; 2529 let Inst{9-5} = Zn; 2530 let Inst{4-0} = Zd; 2531} 2532 2533multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2534 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2535 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2536 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2537 2538 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2539 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2540 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2541} 2542 2543//===----------------------------------------------------------------------===// 2544// SVE Integer Arithmetic - Binary Predicated Group 2545//===----------------------------------------------------------------------===// 2546 2547class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2548 string asm, ZPRRegOp zprty> 2549: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2550 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2551 bits<3> Pg; 2552 bits<5> Zdn; 2553 bits<5> Zm; 2554 let Inst{31-24} = 0b00000100; 2555 let Inst{23-22} = sz8_64; 2556 let Inst{21} = 0b0; 2557 let Inst{20-19} = fmt; 2558 let Inst{18-16} = opc; 2559 let Inst{15-13} = 0b000; 2560 let Inst{12-10} = Pg; 2561 let Inst{9-5} = Zm; 2562 let Inst{4-0} = Zdn; 2563 2564 let Constraints = "$Zdn = $_Zdn"; 2565 let DestructiveInstType = DestructiveOther; 2566 let ElementSize = zprty.ElementSize; 2567} 2568 2569multiclass sve_int_bin_pred_log<bits<3> opc, string asm, SDPatternOperator op> { 2570 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>; 2571 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>; 2572 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>; 2573 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>; 2574 2575 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2576 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2577 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2578 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2579} 2580 2581multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2582 SDPatternOperator op, 2583 DestructiveInstTypeEnum flags, 2584 string revname="", bit isReverseInstr=0> { 2585 let DestructiveInstType = flags in { 2586 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2587 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2588 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2589 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2590 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2591 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2592 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2593 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2594 } 2595 2596 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2597 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2598 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2599 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2600} 2601 2602multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2603 SDPatternOperator op, 2604 DestructiveInstTypeEnum flags> { 2605 let DestructiveInstType = flags in { 2606 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2607 SVEPseudo2Instr<Ps # _B, 1>; 2608 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2609 SVEPseudo2Instr<Ps # _H, 1>; 2610 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2611 SVEPseudo2Instr<Ps # _S, 1>; 2612 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2613 SVEPseudo2Instr<Ps # _D, 1>; 2614 } 2615 2616 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2617 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2618 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2619 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2620} 2621 2622multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2623 SDPatternOperator op, 2624 DestructiveInstTypeEnum flags> { 2625 let DestructiveInstType = flags in { 2626 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2627 SVEPseudo2Instr<Ps # _B, 1>; 2628 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2629 SVEPseudo2Instr<Ps # _H, 1>; 2630 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2631 SVEPseudo2Instr<Ps # _S, 1>; 2632 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2633 SVEPseudo2Instr<Ps # _D, 1>; 2634 } 2635 2636 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2637 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2638 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2639 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2640} 2641 2642// Special case for divides which are not defined for 8b/16b elements. 2643multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2644 SDPatternOperator op, 2645 DestructiveInstTypeEnum flags, 2646 string revname="", bit isReverseInstr=0> { 2647 let DestructiveInstType = flags in { 2648 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2649 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2650 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2651 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2652 } 2653 2654 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2655 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2656} 2657 2658//===----------------------------------------------------------------------===// 2659// SVE Integer Multiply-Add Group 2660//===----------------------------------------------------------------------===// 2661 2662class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2663 ZPRRegOp zprty> 2664: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2665 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2666 "", 2667 []>, Sched<[]> { 2668 bits<3> Pg; 2669 bits<5> Zdn; 2670 bits<5> Za; 2671 bits<5> Zm; 2672 let Inst{31-24} = 0b00000100; 2673 let Inst{23-22} = sz8_64; 2674 let Inst{21} = 0b0; 2675 let Inst{20-16} = Zm; 2676 let Inst{15-14} = 0b11; 2677 let Inst{13} = opc; 2678 let Inst{12-10} = Pg; 2679 let Inst{9-5} = Za; 2680 let Inst{4-0} = Zdn; 2681 2682 let Constraints = "$Zdn = $_Zdn"; 2683 let DestructiveInstType = DestructiveOther; 2684 let ElementSize = zprty.ElementSize; 2685} 2686 2687multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2688 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2689 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2690 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2691 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 2692 2693 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2694 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2695 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2696 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2697} 2698 2699class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2700 ZPRRegOp zprty> 2701: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2702 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2703 "", 2704 []>, Sched<[]> { 2705 bits<3> Pg; 2706 bits<5> Zda; 2707 bits<5> Zm; 2708 bits<5> Zn; 2709 let Inst{31-24} = 0b00000100; 2710 let Inst{23-22} = sz8_64; 2711 let Inst{21} = 0b0; 2712 let Inst{20-16} = Zm; 2713 let Inst{15-14} = 0b01; 2714 let Inst{13} = opc; 2715 let Inst{12-10} = Pg; 2716 let Inst{9-5} = Zn; 2717 let Inst{4-0} = Zda; 2718 2719 let Constraints = "$Zda = $_Zda"; 2720 let DestructiveInstType = DestructiveOther; 2721 let ElementSize = zprty.ElementSize; 2722} 2723 2724multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op, 2725 SDPatternOperator outerop, SDPatternOperator mulop> { 2726 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 2727 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 2728 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 2729 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 2730 2731 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2732 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2733 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2734 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2735 2736 def : Pat<(outerop nxv16i8:$Op1, (mulop nxv16i1:$pred, nxv16i8:$Op2, nxv16i8:$Op3)), 2737 (!cast<Instruction>(NAME # _B) $pred, $Op1, $Op2, $Op3)>; 2738 def : Pat<(outerop nxv8i16:$Op1, (mulop nxv8i1:$pred, nxv8i16:$Op2, nxv8i16:$Op3)), 2739 (!cast<Instruction>(NAME # _H) $pred, $Op1, $Op2, $Op3)>; 2740 def : Pat<(outerop nxv4i32:$Op1, (mulop nxv4i1:$pred, nxv4i32:$Op2, nxv4i32:$Op3)), 2741 (!cast<Instruction>(NAME # _S) $pred, $Op1, $Op2, $Op3)>; 2742 def : Pat<(outerop nxv2i64:$Op1, (mulop nxv2i1:$pred, nxv2i64:$Op2, nxv2i64:$Op3)), 2743 (!cast<Instruction>(NAME # _D) $pred, $Op1, $Op2, $Op3)>; 2744} 2745 2746//===----------------------------------------------------------------------===// 2747// SVE2 Integer Multiply-Add - Unpredicated Group 2748//===----------------------------------------------------------------------===// 2749 2750class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 2751 ZPRRegOp zprty1, ZPRRegOp zprty2> 2752: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 2753 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2754 bits<5> Zda; 2755 bits<5> Zn; 2756 bits<5> Zm; 2757 let Inst{31-24} = 0b01000100; 2758 let Inst{23-22} = sz; 2759 let Inst{21} = 0b0; 2760 let Inst{20-16} = Zm; 2761 let Inst{15} = 0b0; 2762 let Inst{14-10} = opc; 2763 let Inst{9-5} = Zn; 2764 let Inst{4-0} = Zda; 2765 2766 let Constraints = "$Zda = $_Zda"; 2767 let DestructiveInstType = DestructiveOther; 2768 let ElementSize = ElementSizeNone; 2769} 2770 2771multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 2772 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 2773 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 2774 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 2775 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 2776 2777 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2778 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2779 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2780 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2781} 2782 2783multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 2784 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 2785 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 2786 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 2787 2788 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 2789 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 2790 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 2791} 2792 2793//===----------------------------------------------------------------------===// 2794// SVE2 Integer Multiply-Add - Indexed Group 2795//===----------------------------------------------------------------------===// 2796 2797class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 2798 ZPRRegOp zprty1, ZPRRegOp zprty2, 2799 ZPRRegOp zprty3, Operand itype> 2800: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2801 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2802 bits<5> Zda; 2803 bits<5> Zn; 2804 let Inst{31-24} = 0b01000100; 2805 let Inst{23-22} = sz; 2806 let Inst{21} = 0b1; 2807 let Inst{15-10} = opc; 2808 let Inst{9-5} = Zn; 2809 let Inst{4-0} = Zda; 2810 2811 let Constraints = "$Zda = $_Zda"; 2812 let DestructiveInstType = DestructiveOther; 2813 let ElementSize = ElementSizeNone; 2814} 2815 2816multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 2817 SDPatternOperator op> { 2818 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 2819 bits<3> Zm; 2820 bits<3> iop; 2821 let Inst{22} = iop{2}; 2822 let Inst{20-19} = iop{1-0}; 2823 let Inst{18-16} = Zm; 2824 } 2825 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 2826 bits<3> Zm; 2827 bits<2> iop; 2828 let Inst{20-19} = iop; 2829 let Inst{18-16} = Zm; 2830 } 2831 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 2832 bits<4> Zm; 2833 bit iop; 2834 let Inst{20} = iop; 2835 let Inst{19-16} = Zm; 2836 } 2837 2838 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 2839 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 2840 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 2841} 2842 2843//===----------------------------------------------------------------------===// 2844// SVE2 Integer Multiply-Add Long - Indexed Group 2845//===----------------------------------------------------------------------===// 2846 2847multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 2848 SDPatternOperator op> { 2849 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2850 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 2851 bits<3> Zm; 2852 bits<3> iop; 2853 let Inst{20-19} = iop{2-1}; 2854 let Inst{18-16} = Zm; 2855 let Inst{11} = iop{0}; 2856 } 2857 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 2858 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 2859 bits<4> Zm; 2860 bits<2> iop; 2861 let Inst{20} = iop{1}; 2862 let Inst{19-16} = Zm; 2863 let Inst{11} = iop{0}; 2864 } 2865 2866 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 2867 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 2868} 2869 2870//===----------------------------------------------------------------------===// 2871// SVE Integer Dot Product Group 2872//===----------------------------------------------------------------------===// 2873 2874class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 2875 ZPRRegOp zprty2> 2876: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 2877 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 2878 bits<5> Zda; 2879 bits<5> Zn; 2880 bits<5> Zm; 2881 let Inst{31-23} = 0b010001001; 2882 let Inst{22} = sz; 2883 let Inst{21} = 0; 2884 let Inst{20-16} = Zm; 2885 let Inst{15-11} = 0; 2886 let Inst{10} = U; 2887 let Inst{9-5} = Zn; 2888 let Inst{4-0} = Zda; 2889 2890 let Constraints = "$Zda = $_Zda"; 2891 let DestructiveInstType = DestructiveOther; 2892} 2893 2894multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 2895 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 2896 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 2897 2898 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 2899 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 2900} 2901 2902//===----------------------------------------------------------------------===// 2903// SVE Integer Dot Product Group - Indexed Group 2904//===----------------------------------------------------------------------===// 2905 2906class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 2907 ZPRRegOp zprty1, ZPRRegOp zprty2, 2908 ZPRRegOp zprty3, Operand itype> 2909: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 2910 asm, "\t$Zda, $Zn, $Zm$iop", 2911 "", []>, Sched<[]> { 2912 bits<5> Zda; 2913 bits<5> Zn; 2914 let Inst{31-23} = 0b010001001; 2915 let Inst{22} = sz; 2916 let Inst{21} = 0b1; 2917 let Inst{15-11} = 0; 2918 let Inst{10} = U; 2919 let Inst{9-5} = Zn; 2920 let Inst{4-0} = Zda; 2921 2922 let Constraints = "$Zda = $_Zda"; 2923 let DestructiveInstType = DestructiveOther; 2924} 2925 2926multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 2927 SDPatternOperator op> { 2928 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 2929 bits<2> iop; 2930 bits<3> Zm; 2931 let Inst{20-19} = iop; 2932 let Inst{18-16} = Zm; 2933 } 2934 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 2935 bits<1> iop; 2936 bits<4> Zm; 2937 let Inst{20} = iop; 2938 let Inst{19-16} = Zm; 2939 } 2940 2941 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 2942 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 2943} 2944 2945//===----------------------------------------------------------------------===// 2946// SVE2 Complex Integer Dot Product Group 2947//===----------------------------------------------------------------------===// 2948 2949class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 2950 ZPRRegOp zprty1, ZPRRegOp zprty2> 2951: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 2952 complexrotateop:$rot), 2953 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 2954 bits<5> Zda; 2955 bits<5> Zn; 2956 bits<5> Zm; 2957 bits<2> rot; 2958 let Inst{31-24} = 0b01000100; 2959 let Inst{23-22} = sz; 2960 let Inst{21} = 0b0; 2961 let Inst{20-16} = Zm; 2962 let Inst{15-12} = opc; 2963 let Inst{11-10} = rot; 2964 let Inst{9-5} = Zn; 2965 let Inst{4-0} = Zda; 2966 2967 let Constraints = "$Zda = $_Zda"; 2968 let DestructiveInstType = DestructiveOther; 2969 let ElementSize = ElementSizeNone; 2970} 2971 2972multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 2973 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 2974 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 2975 2976 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 2977 (i32 complexrotateop:$imm))), 2978 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 2979 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 2980 (i32 complexrotateop:$imm))), 2981 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 2982} 2983 2984//===----------------------------------------------------------------------===// 2985// SVE2 Complex Multiply-Add Group 2986//===----------------------------------------------------------------------===// 2987 2988multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 2989 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 2990 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 2991 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 2992 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 2993 2994 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 2995 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 2996 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 2997 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 2998} 2999 3000//===----------------------------------------------------------------------===// 3001// SVE2 Complex Integer Dot Product - Indexed Group 3002//===----------------------------------------------------------------------===// 3003 3004class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 3005 ZPRRegOp zprty1, ZPRRegOp zprty2, 3006 ZPRRegOp zprty3, Operand itype> 3007: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 3008 complexrotateop:$rot), 3009 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 3010 bits<5> Zda; 3011 bits<5> Zn; 3012 bits<2> rot; 3013 let Inst{31-24} = 0b01000100; 3014 let Inst{23-22} = sz; 3015 let Inst{21} = 0b1; 3016 let Inst{15-12} = opc; 3017 let Inst{11-10} = rot; 3018 let Inst{9-5} = Zn; 3019 let Inst{4-0} = Zda; 3020 3021 let Constraints = "$Zda = $_Zda"; 3022 let DestructiveInstType = DestructiveOther; 3023 let ElementSize = ElementSizeNone; 3024} 3025 3026multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 3027 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 3028 bits<2> iop; 3029 bits<3> Zm; 3030 let Inst{20-19} = iop; 3031 let Inst{18-16} = Zm; 3032 } 3033 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 3034 bit iop; 3035 bits<4> Zm; 3036 let Inst{20} = iop; 3037 let Inst{19-16} = Zm; 3038 } 3039 3040 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3041 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3042 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3043 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3044 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3045 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3046} 3047 3048//===----------------------------------------------------------------------===// 3049// SVE2 Complex Multiply-Add - Indexed Group 3050//===----------------------------------------------------------------------===// 3051 3052multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 3053 SDPatternOperator op> { 3054 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 3055 bits<2> iop; 3056 bits<3> Zm; 3057 let Inst{20-19} = iop; 3058 let Inst{18-16} = Zm; 3059 } 3060 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 3061 bit iop; 3062 bits<4> Zm; 3063 let Inst{20} = iop; 3064 let Inst{19-16} = Zm; 3065 } 3066 3067 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3068 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3069 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3070 3071 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 3072 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3073 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3074} 3075 3076//===----------------------------------------------------------------------===// 3077// SVE2 Integer Multiply - Unpredicated Group 3078//===----------------------------------------------------------------------===// 3079 3080class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 3081: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 3082 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3083 bits<5> Zd; 3084 bits<5> Zm; 3085 bits<5> Zn; 3086 let Inst{31-24} = 0b00000100; 3087 let Inst{23-22} = sz; 3088 let Inst{21} = 0b1; 3089 let Inst{20-16} = Zm; 3090 let Inst{15-13} = 0b011; 3091 let Inst{12-10} = opc; 3092 let Inst{9-5} = Zn; 3093 let Inst{4-0} = Zd; 3094} 3095 3096multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 3097 SDPatternOperator op_pred = null_frag> { 3098 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3099 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 3100 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 3101 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 3102 3103 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3104 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3105 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3106 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3107 3108 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3109 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3110 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3111 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3112} 3113 3114multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 3115 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3116 3117 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3118} 3119 3120//===----------------------------------------------------------------------===// 3121// SVE2 Integer Multiply - Indexed Group 3122//===----------------------------------------------------------------------===// 3123 3124class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 3125 ZPRRegOp zprty1, ZPRRegOp zprty2, 3126 ZPRRegOp zprty3, Operand itype> 3127: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 3128 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 3129 bits<5> Zd; 3130 bits<5> Zn; 3131 let Inst{31-24} = 0b01000100; 3132 let Inst{23-22} = sz; 3133 let Inst{21} = 0b1; 3134 let Inst{15-14} = 0b11; 3135 let Inst{13-10} = opc; 3136 let Inst{9-5} = Zn; 3137 let Inst{4-0} = Zd; 3138} 3139 3140multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3141 SDPatternOperator op> { 3142 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3143 bits<3> Zm; 3144 bits<3> iop; 3145 let Inst{22} = iop{2}; 3146 let Inst{20-19} = iop{1-0}; 3147 let Inst{18-16} = Zm; 3148 } 3149 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3150 bits<3> Zm; 3151 bits<2> iop; 3152 let Inst{20-19} = iop; 3153 let Inst{18-16} = Zm; 3154 } 3155 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3156 bits<4> Zm; 3157 bit iop; 3158 let Inst{20} = iop; 3159 let Inst{19-16} = Zm; 3160 } 3161 3162 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3163 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3164 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3165} 3166 3167multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3168 SDPatternOperator op> { 3169 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3170 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3171 bits<3> Zm; 3172 bits<3> iop; 3173 let Inst{20-19} = iop{2-1}; 3174 let Inst{18-16} = Zm; 3175 let Inst{11} = iop{0}; 3176 } 3177 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3178 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3179 bits<4> Zm; 3180 bits<2> iop; 3181 let Inst{20} = iop{1}; 3182 let Inst{19-16} = Zm; 3183 let Inst{11} = iop{0}; 3184 } 3185 3186 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3187 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3188} 3189 3190//===----------------------------------------------------------------------===// 3191// SVE2 Integer - Predicated Group 3192//===----------------------------------------------------------------------===// 3193 3194class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3195 ZPRRegOp zprty> 3196: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3197 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3198 bits<3> Pg; 3199 bits<5> Zm; 3200 bits<5> Zdn; 3201 let Inst{31-24} = 0b01000100; 3202 let Inst{23-22} = sz; 3203 let Inst{21-20} = 0b01; 3204 let Inst{20-16} = opc{5-1}; 3205 let Inst{15-14} = 0b10; 3206 let Inst{13} = opc{0}; 3207 let Inst{12-10} = Pg; 3208 let Inst{9-5} = Zm; 3209 let Inst{4-0} = Zdn; 3210 3211 let Constraints = "$Zdn = $_Zdn"; 3212 let DestructiveInstType = DestructiveOther; 3213 let ElementSize = zprty.ElementSize; 3214} 3215 3216multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op, 3217 string Ps = "", 3218 DestructiveInstTypeEnum flags=DestructiveOther, 3219 string revname="", bit isReverseInstr=0> { 3220 let DestructiveInstType = flags in { 3221 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>, 3222 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 3223 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>, 3224 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 3225 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>, 3226 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 3227 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>, 3228 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 3229 } 3230 3231 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3232 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3233 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3234 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3235} 3236 3237class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3238 ZPRRegOp zprty1, ZPRRegOp zprty2> 3239: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3240 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3241 bits<3> Pg; 3242 bits<5> Zn; 3243 bits<5> Zda; 3244 let Inst{31-24} = 0b01000100; 3245 let Inst{23-22} = sz; 3246 let Inst{21-17} = 0b00010; 3247 let Inst{16} = U; 3248 let Inst{15-13} = 0b101; 3249 let Inst{12-10} = Pg; 3250 let Inst{9-5} = Zn; 3251 let Inst{4-0} = Zda; 3252 3253 let Constraints = "$Zda = $_Zda"; 3254 let DestructiveInstType = DestructiveOther; 3255 let ElementSize = zprty1.ElementSize; 3256} 3257 3258multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3259 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3260 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3261 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3262 3263 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3264 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3265 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3266} 3267 3268class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3269 string asm, ZPRRegOp zprty> 3270: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3271 asm, "\t$Zd, $Pg/m, $Zn", 3272 "", 3273 []>, Sched<[]> { 3274 bits<3> Pg; 3275 bits<5> Zd; 3276 bits<5> Zn; 3277 let Inst{31-24} = 0b01000100; 3278 let Inst{23-22} = sz; 3279 let Inst{21-20} = 0b00; 3280 let Inst{19} = Q; 3281 let Inst{18} = 0b0; 3282 let Inst{17-16} = opc; 3283 let Inst{15-13} = 0b101; 3284 let Inst{12-10} = Pg; 3285 let Inst{9-5} = Zn; 3286 let Inst{4-0} = Zd; 3287 3288 let Constraints = "$Zd = $_Zd"; 3289 let DestructiveInstType = DestructiveUnaryPassthru; 3290 let ElementSize = zprty.ElementSize; 3291} 3292 3293multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3294 SDPatternOperator op> { 3295 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3296 SVEPseudo2Instr<NAME # _S, 1>; 3297 3298 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3299 3300 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3301 3302 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3303} 3304 3305multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3306 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>, 3307 SVEPseudo2Instr<NAME # _B, 1>; 3308 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>, 3309 SVEPseudo2Instr<NAME # _H, 1>; 3310 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3311 SVEPseudo2Instr<NAME # _S, 1>; 3312 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>, 3313 SVEPseudo2Instr<NAME # _D, 1>; 3314 3315 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3316 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3317 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3318 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3319 3320 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3321 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3322 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3323 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3324 3325 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3326 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3327 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3328 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3329} 3330 3331//===----------------------------------------------------------------------===// 3332// SVE2 Widening Integer Arithmetic Group 3333//===----------------------------------------------------------------------===// 3334 3335class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3336 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3337: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3338 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3339 bits<5> Zd; 3340 bits<5> Zn; 3341 bits<5> Zm; 3342 let Inst{31-24} = 0b01000101; 3343 let Inst{23-22} = sz; 3344 let Inst{21} = 0b0; 3345 let Inst{20-16} = Zm; 3346 let Inst{15} = 0b0; 3347 let Inst{14-10} = opc; 3348 let Inst{9-5} = Zn; 3349 let Inst{4-0} = Zd; 3350} 3351 3352multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3353 SDPatternOperator op> { 3354 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3355 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3356 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3357 3358 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3359 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3360 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3361} 3362 3363multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3364 SDPatternOperator op> { 3365 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3366 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3367 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3368 3369 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3370 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3371 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3372} 3373 3374multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3375 SDPatternOperator op> { 3376 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3377 3378 // To avoid using 128 bit elements in the IR, the pattern below works with 3379 // llvm intrinsics with the _pair suffix, to reflect that 3380 // _Q is implemented as a pair of _D. 3381 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3382} 3383 3384multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3385 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3386 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3387 3388 // To avoid using 128 bit elements in the IR, the patterns below work with 3389 // llvm intrinsics with the _pair suffix, to reflect that 3390 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3391 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3392 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3393} 3394 3395//===----------------------------------------------------------------------===// 3396// SVE2 Misc Group 3397//===----------------------------------------------------------------------===// 3398 3399class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3400 ZPRRegOp zprty1, ZPRRegOp zprty2> 3401: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3402 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3403 bits<5> Zd; 3404 bits<5> Zn; 3405 bits<5> Zm; 3406 let Inst{31-24} = 0b01000101; 3407 let Inst{23-22} = sz; 3408 let Inst{21} = 0b0; 3409 let Inst{20-16} = Zm; 3410 let Inst{15-14} = 0b10; 3411 let Inst{13-10} = opc; 3412 let Inst{9-5} = Zn; 3413 let Inst{4-0} = Zd; 3414} 3415 3416multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3417 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3418 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3419 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3420 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3421 3422 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3423 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3424 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3425 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3426} 3427 3428multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3429 SDPatternOperator op> { 3430 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3431 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3432 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3433 3434 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3435 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3436 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3437} 3438 3439class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3440 ZPRRegOp zprty1, ZPRRegOp zprty2> 3441: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3442 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3443 bits<5> Zd; 3444 bits<5> Zn; 3445 bits<5> Zm; 3446 let Inst{31-24} = 0b01000101; 3447 let Inst{23-22} = sz; 3448 let Inst{21} = 0b0; 3449 let Inst{20-16} = Zm; 3450 let Inst{15-11} = 0b10010; 3451 let Inst{10} = opc; 3452 let Inst{9-5} = Zn; 3453 let Inst{4-0} = Zd; 3454 3455 let Constraints = "$Zd = $_Zd"; 3456 let DestructiveInstType = DestructiveOther; 3457 let ElementSize = ElementSizeNone; 3458} 3459 3460multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3461 SDPatternOperator op> { 3462 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3463 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3464 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3465 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3466 3467 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3468 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3469 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3470 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3471} 3472 3473class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3474 ZPRRegOp zprty1, ZPRRegOp zprty2, 3475 Operand immtype> 3476: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3477 asm, "\t$Zd, $Zn, $imm", 3478 "", []>, Sched<[]> { 3479 bits<5> Zd; 3480 bits<5> Zn; 3481 bits<5> imm; 3482 let Inst{31-23} = 0b010001010; 3483 let Inst{22} = tsz8_64{2}; 3484 let Inst{21} = 0b0; 3485 let Inst{20-19} = tsz8_64{1-0}; 3486 let Inst{18-16} = imm{2-0}; // imm3 3487 let Inst{15-12} = 0b1010; 3488 let Inst{11-10} = opc; 3489 let Inst{9-5} = Zn; 3490 let Inst{4-0} = Zd; 3491} 3492 3493multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3494 SDPatternOperator op> { 3495 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3496 ZPR16, ZPR8, vecshiftL8>; 3497 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3498 ZPR32, ZPR16, vecshiftL16> { 3499 let Inst{19} = imm{3}; 3500 } 3501 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3502 ZPR64, ZPR32, vecshiftL32> { 3503 let Inst{20-19} = imm{4-3}; 3504 } 3505 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3506 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3507 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3508} 3509 3510//===----------------------------------------------------------------------===// 3511// SVE2 Accumulate Group 3512//===----------------------------------------------------------------------===// 3513 3514class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3515 ZPRRegOp zprty, Operand immtype> 3516: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3517 asm, "\t$Zd, $Zn, $imm", 3518 "", []>, Sched<[]> { 3519 bits<5> Zd; 3520 bits<5> Zn; 3521 bits<6> imm; 3522 let Inst{31-24} = 0b01000101; 3523 let Inst{23-22} = tsz8_64{3-2}; 3524 let Inst{21} = 0b0; 3525 let Inst{20-19} = tsz8_64{1-0}; 3526 let Inst{18-16} = imm{2-0}; // imm3 3527 let Inst{15-11} = 0b11110; 3528 let Inst{10} = opc; 3529 let Inst{9-5} = Zn; 3530 let Inst{4-0} = Zd; 3531 3532 let Constraints = "$Zd = $_Zd"; 3533} 3534 3535multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3536 SDPatternOperator op> { 3537 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3538 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3539 let Inst{19} = imm{3}; 3540 } 3541 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3542 let Inst{20-19} = imm{4-3}; 3543 } 3544 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3545 let Inst{22} = imm{5}; 3546 let Inst{20-19} = imm{4-3}; 3547 } 3548 3549 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3550 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3551 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3552 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3553} 3554 3555multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3556 SDPatternOperator op> { 3557 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3558 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3559 let Inst{19} = imm{3}; 3560 } 3561 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3562 let Inst{20-19} = imm{4-3}; 3563 } 3564 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3565 let Inst{22} = imm{5}; 3566 let Inst{20-19} = imm{4-3}; 3567 } 3568 3569 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3570 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3571 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3572 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3573} 3574 3575class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3576 ZPRRegOp zprty, Operand immtype> 3577: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3578 asm, "\t$Zda, $Zn, $imm", 3579 "", []>, Sched<[]> { 3580 bits<5> Zda; 3581 bits<5> Zn; 3582 bits<6> imm; 3583 let Inst{31-24} = 0b01000101; 3584 let Inst{23-22} = tsz8_64{3-2}; 3585 let Inst{21} = 0b0; 3586 let Inst{20-19} = tsz8_64{1-0}; 3587 let Inst{18-16} = imm{2-0}; // imm3 3588 let Inst{15-12} = 0b1110; 3589 let Inst{11-10} = opc; 3590 let Inst{9-5} = Zn; 3591 let Inst{4-0} = Zda; 3592 3593 let Constraints = "$Zda = $_Zda"; 3594 let DestructiveInstType = DestructiveOther; 3595 let ElementSize = ElementSizeNone; 3596} 3597 3598multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3599 SDPatternOperator op> { 3600 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3601 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3602 let Inst{19} = imm{3}; 3603 } 3604 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3605 let Inst{20-19} = imm{4-3}; 3606 } 3607 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3608 let Inst{22} = imm{5}; 3609 let Inst{20-19} = imm{4-3}; 3610 } 3611 3612 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3613 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3614 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3615 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3616} 3617 3618class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3619: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3620 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3621 bits<5> Zdn; 3622 bits<5> Zm; 3623 bit rot; 3624 let Inst{31-24} = 0b01000101; 3625 let Inst{23-22} = sz; 3626 let Inst{21-17} = 0b00000; 3627 let Inst{16} = opc; 3628 let Inst{15-11} = 0b11011; 3629 let Inst{10} = rot; 3630 let Inst{9-5} = Zm; 3631 let Inst{4-0} = Zdn; 3632 3633 let Constraints = "$Zdn = $_Zdn"; 3634 let DestructiveInstType = DestructiveOther; 3635 let ElementSize = ElementSizeNone; 3636} 3637 3638multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3639 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3640 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3641 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3642 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3643 3644 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3645 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3646 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3647 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3648} 3649 3650class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3651 ZPRRegOp zprty1, ZPRRegOp zprty2> 3652: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3653 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3654 bits<5> Zda; 3655 bits<5> Zn; 3656 bits<5> Zm; 3657 let Inst{31-24} = 0b01000101; 3658 let Inst{23-22} = sz; 3659 let Inst{21} = 0b0; 3660 let Inst{20-16} = Zm; 3661 let Inst{15-14} = 0b11; 3662 let Inst{13-10} = opc; 3663 let Inst{9-5} = Zn; 3664 let Inst{4-0} = Zda; 3665 3666 let Constraints = "$Zda = $_Zda"; 3667 let DestructiveInstType = DestructiveOther; 3668 let ElementSize = ElementSizeNone; 3669} 3670 3671multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3672 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3673 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3674 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3675 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3676 3677 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3678 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3679 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3680 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3681} 3682 3683multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3684 SDPatternOperator op> { 3685 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3686 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3687 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3688 3689 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3690 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3691 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3692} 3693 3694multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3695 SDPatternOperator op> { 3696 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 3697 ZPR32, ZPR32>; 3698 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 3699 ZPR64, ZPR64>; 3700 3701 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3702 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3703} 3704 3705//===----------------------------------------------------------------------===// 3706// SVE2 Narrowing Group 3707//===----------------------------------------------------------------------===// 3708 3709class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 3710 string asm, ZPRRegOp zprty1, 3711 ZPRRegOp zprty2, Operand immtype> 3712: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3713 asm, "\t$Zd, $Zn, $imm", 3714 "", []>, Sched<[]> { 3715 bits<5> Zd; 3716 bits<5> Zn; 3717 bits<5> imm; 3718 let Inst{31-23} = 0b010001010; 3719 let Inst{22} = tsz8_64{2}; 3720 let Inst{21} = 0b1; 3721 let Inst{20-19} = tsz8_64{1-0}; 3722 let Inst{18-16} = imm{2-0}; // imm3 3723 let Inst{15-14} = 0b00; 3724 let Inst{13-11} = opc; 3725 let Inst{10} = 0b0; 3726 let Inst{9-5} = Zn; 3727 let Inst{4-0} = Zd; 3728} 3729 3730multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 3731 SDPatternOperator op> { 3732 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 3733 tvecshiftR8>; 3734 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 3735 tvecshiftR16> { 3736 let Inst{19} = imm{3}; 3737 } 3738 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 3739 tvecshiftR32> { 3740 let Inst{20-19} = imm{4-3}; 3741 } 3742 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3743 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3744 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3745} 3746 3747class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 3748 string asm, ZPRRegOp zprty1, 3749 ZPRRegOp zprty2, Operand immtype> 3750: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 3751 asm, "\t$Zd, $Zn, $imm", 3752 "", []>, Sched<[]> { 3753 bits<5> Zd; 3754 bits<5> Zn; 3755 bits<5> imm; 3756 let Inst{31-23} = 0b010001010; 3757 let Inst{22} = tsz8_64{2}; 3758 let Inst{21} = 0b1; 3759 let Inst{20-19} = tsz8_64{1-0}; 3760 let Inst{18-16} = imm{2-0}; // imm3 3761 let Inst{15-14} = 0b00; 3762 let Inst{13-11} = opc; 3763 let Inst{10} = 0b1; 3764 let Inst{9-5} = Zn; 3765 let Inst{4-0} = Zd; 3766 3767 let Constraints = "$Zd = $_Zd"; 3768} 3769 3770multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 3771 SDPatternOperator op> { 3772 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 3773 tvecshiftR8>; 3774 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 3775 tvecshiftR16> { 3776 let Inst{19} = imm{3}; 3777 } 3778 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 3779 tvecshiftR32> { 3780 let Inst{20-19} = imm{4-3}; 3781 } 3782 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3783 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3784 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3785} 3786 3787class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 3788 ZPRRegOp zprty1, ZPRRegOp zprty2> 3789: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3790 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3791 bits<5> Zd; 3792 bits<5> Zn; 3793 bits<5> Zm; 3794 let Inst{31-24} = 0b01000101; 3795 let Inst{23-22} = sz; 3796 let Inst{21} = 0b1; 3797 let Inst{20-16} = Zm; 3798 let Inst{15-13} = 0b011; 3799 let Inst{12-11} = opc; // S, R 3800 let Inst{10} = 0b0; // Top 3801 let Inst{9-5} = Zn; 3802 let Inst{4-0} = Zd; 3803} 3804 3805multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 3806 SDPatternOperator op> { 3807 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 3808 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 3809 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 3810 3811 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3812 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3813 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3814} 3815 3816class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 3817 ZPRRegOp zprty1, ZPRRegOp zprty2> 3818: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3819 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3820 bits<5> Zd; 3821 bits<5> Zn; 3822 bits<5> Zm; 3823 let Inst{31-24} = 0b01000101; 3824 let Inst{23-22} = sz; 3825 let Inst{21} = 0b1; 3826 let Inst{20-16} = Zm; 3827 let Inst{15-13} = 0b011; 3828 let Inst{12-11} = opc; // S, R 3829 let Inst{10} = 0b1; // Top 3830 let Inst{9-5} = Zn; 3831 let Inst{4-0} = Zd; 3832 3833 let Constraints = "$Zd = $_Zd"; 3834} 3835 3836multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 3837 SDPatternOperator op> { 3838 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 3839 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 3840 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 3841 3842 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 3843 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 3844 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 3845} 3846 3847class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 3848 ZPRRegOp zprty1, ZPRRegOp zprty2> 3849: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 3850 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3851 bits<5> Zd; 3852 bits<5> Zn; 3853 let Inst{31-23} = 0b010001010; 3854 let Inst{22} = tsz8_64{2}; 3855 let Inst{21} = 0b1; 3856 let Inst{20-19} = tsz8_64{1-0}; 3857 let Inst{18-13} = 0b000010; 3858 let Inst{12-11} = opc; 3859 let Inst{10} = 0b0; 3860 let Inst{9-5} = Zn; 3861 let Inst{4-0} = Zd; 3862} 3863 3864multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 3865 SDPatternOperator op> { 3866 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 3867 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 3868 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 3869 3870 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 3871 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 3872 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 3873} 3874 3875class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 3876 ZPRRegOp zprty1, ZPRRegOp zprty2> 3877: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 3878 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 3879 bits<5> Zd; 3880 bits<5> Zn; 3881 let Inst{31-23} = 0b010001010; 3882 let Inst{22} = tsz8_64{2}; 3883 let Inst{21} = 0b1; 3884 let Inst{20-19} = tsz8_64{1-0}; 3885 let Inst{18-13} = 0b000010; 3886 let Inst{12-11} = opc; 3887 let Inst{10} = 0b1; 3888 let Inst{9-5} = Zn; 3889 let Inst{4-0} = Zd; 3890 3891 let Constraints = "$Zd = $_Zd"; 3892} 3893 3894multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 3895 SDPatternOperator op> { 3896 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 3897 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 3898 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 3899 3900 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 3901 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 3902 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 3903} 3904 3905//===----------------------------------------------------------------------===// 3906// SVE Integer Arithmetic - Unary Predicated Group 3907//===----------------------------------------------------------------------===// 3908 3909class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 3910 string asm, ZPRRegOp zprty> 3911: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3912 asm, "\t$Zd, $Pg/m, $Zn", 3913 "", 3914 []>, Sched<[]> { 3915 bits<3> Pg; 3916 bits<5> Zd; 3917 bits<5> Zn; 3918 let Inst{31-24} = 0b00000100; 3919 let Inst{23-22} = sz8_64; 3920 let Inst{21-20} = 0b01; 3921 let Inst{19} = opc{0}; 3922 let Inst{18-16} = opc{3-1}; 3923 let Inst{15-13} = 0b101; 3924 let Inst{12-10} = Pg; 3925 let Inst{9-5} = Zn; 3926 let Inst{4-0} = Zd; 3927 3928 let Constraints = "$Zd = $_Zd"; 3929 let DestructiveInstType = DestructiveUnaryPassthru; 3930 let ElementSize = zprty.ElementSize; 3931} 3932 3933multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 3934 SDPatternOperator op> { 3935 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>, 3936 SVEPseudo2Instr<NAME # _B, 1>; 3937 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 3938 SVEPseudo2Instr<NAME # _H, 1>; 3939 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 3940 SVEPseudo2Instr<NAME # _S, 1>; 3941 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 3942 SVEPseudo2Instr<NAME # _D, 1>; 3943 3944 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3945 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3946 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3947 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3948 3949 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3950 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3951 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3952 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3953 3954 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3955 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3956 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3957 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3958} 3959 3960multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 3961 SDPatternOperator op> { 3962 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 3963 SVEPseudo2Instr<NAME # _H, 1>; 3964 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 3965 SVEPseudo2Instr<NAME # _S, 1>; 3966 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 3967 SVEPseudo2Instr<NAME # _D, 1>; 3968 3969 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 3970 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 3971 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 3972 3973 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3974 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3975 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3976 3977 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _UNDEF_H)>; 3978 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _UNDEF_S)>; 3979 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _UNDEF_D)>; 3980} 3981 3982multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 3983 SDPatternOperator op> { 3984 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 3985 SVEPseudo2Instr<NAME # _S, 1>; 3986 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 3987 SVEPseudo2Instr<NAME # _D, 1>; 3988 3989 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 3990 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 3991 3992 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3993 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3994 3995 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _UNDEF_S)>; 3996 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _UNDEF_D)>; 3997} 3998 3999multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 4000 SDPatternOperator op> { 4001 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4002 SVEPseudo2Instr<NAME # _D, 1>; 4003 4004 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 4005 4006 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4007 4008 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _UNDEF_D)>; 4009} 4010 4011multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 4012 SDPatternOperator op> { 4013 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>, 4014 SVEPseudo2Instr<NAME # _B, 1>; 4015 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4016 SVEPseudo2Instr<NAME # _H, 1>; 4017 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4018 SVEPseudo2Instr<NAME # _S, 1>; 4019 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4020 SVEPseudo2Instr<NAME # _D, 1>; 4021 4022 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4023 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4024 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4025 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4026 4027 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4028 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4029 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4030 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4031 4032 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4033 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4034 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4035 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4036} 4037 4038multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 4039 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4040 SVEPseudo2Instr<NAME # _H, 1>; 4041 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4042 SVEPseudo2Instr<NAME # _S, 1>; 4043 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4044 SVEPseudo2Instr<NAME # _D, 1>; 4045 4046 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4047 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4048 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4049 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4050 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4051 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4052 4053 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4054 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4055 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4056 4057 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4058 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4059 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4060 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4061 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4062 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4063} 4064 4065//===----------------------------------------------------------------------===// 4066// SVE Integer Wide Immediate - Unpredicated Group 4067//===----------------------------------------------------------------------===// 4068class sve_int_dup_imm<bits<2> sz8_64, string asm, 4069 ZPRRegOp zprty, Operand immtype> 4070: I<(outs zprty:$Zd), (ins immtype:$imm), 4071 asm, "\t$Zd, $imm", 4072 "", 4073 []>, Sched<[]> { 4074 bits<5> Zd; 4075 bits<9> imm; 4076 let Inst{31-24} = 0b00100101; 4077 let Inst{23-22} = sz8_64; 4078 let Inst{21-14} = 0b11100011; 4079 let Inst{13} = imm{8}; // sh 4080 let Inst{12-5} = imm{7-0}; // imm8 4081 let Inst{4-0} = Zd; 4082 4083 let isReMaterializable = 1; 4084} 4085 4086multiclass sve_int_dup_imm<string asm> { 4087 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 4088 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 4089 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 4090 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 4091 4092 def : InstAlias<"mov $Zd, $imm", 4093 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 4094 def : InstAlias<"mov $Zd, $imm", 4095 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 4096 def : InstAlias<"mov $Zd, $imm", 4097 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 4098 def : InstAlias<"mov $Zd, $imm", 4099 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 4100 4101 def : InstAlias<"fmov $Zd, #0.0", 4102 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 4103 def : InstAlias<"fmov $Zd, #0.0", 4104 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 4105 def : InstAlias<"fmov $Zd, #0.0", 4106 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 4107} 4108 4109class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 4110 string asm, ZPRRegOp zprty> 4111: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 4112 asm, "\t$Zd, $imm8", 4113 "", 4114 []>, Sched<[]> { 4115 bits<5> Zd; 4116 bits<8> imm8; 4117 let Inst{31-24} = 0b00100101; 4118 let Inst{23-22} = sz8_64; 4119 let Inst{21-14} = 0b11100111; 4120 let Inst{13} = 0b0; 4121 let Inst{12-5} = imm8; 4122 let Inst{4-0} = Zd; 4123 4124 let isReMaterializable = 1; 4125} 4126 4127multiclass sve_int_dup_fpimm<string asm> { 4128 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 4129 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 4130 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 4131 4132 def : InstAlias<"fmov $Zd, $imm8", 4133 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 4134 def : InstAlias<"fmov $Zd, $imm8", 4135 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 4136 def : InstAlias<"fmov $Zd, $imm8", 4137 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 4138} 4139 4140class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 4141 ZPRRegOp zprty, Operand immtype> 4142: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4143 asm, "\t$Zdn, $_Zdn, $imm", 4144 "", 4145 []>, Sched<[]> { 4146 bits<5> Zdn; 4147 bits<9> imm; 4148 let Inst{31-24} = 0b00100101; 4149 let Inst{23-22} = sz8_64; 4150 let Inst{21-19} = 0b100; 4151 let Inst{18-16} = opc; 4152 let Inst{15-14} = 0b11; 4153 let Inst{13} = imm{8}; // sh 4154 let Inst{12-5} = imm{7-0}; // imm8 4155 let Inst{4-0} = Zdn; 4156 4157 let Constraints = "$Zdn = $_Zdn"; 4158 let DestructiveInstType = DestructiveOther; 4159 let ElementSize = ElementSizeNone; 4160} 4161 4162multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { 4163 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4164 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4165 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4166 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4167 4168 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4169 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4170 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4171 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4172} 4173 4174multiclass sve_int_arith_imm0_subr<bits<3> opc, string asm, SDPatternOperator op> { 4175 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4176 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4177 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4178 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4179 4180 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4181 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4182 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4183 def : SVE_1_Op_Imm_OptLsl_Reverse_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4184} 4185 4186class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 4187 ZPRRegOp zprty, Operand immtype> 4188: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4189 asm, "\t$Zdn, $_Zdn, $imm", 4190 "", 4191 []>, Sched<[]> { 4192 bits<5> Zdn; 4193 bits<8> imm; 4194 let Inst{31-24} = 0b00100101; 4195 let Inst{23-22} = sz8_64; 4196 let Inst{21-16} = opc; 4197 let Inst{15-13} = 0b110; 4198 let Inst{12-5} = imm; 4199 let Inst{4-0} = Zdn; 4200 4201 let Constraints = "$Zdn = $_Zdn"; 4202 let DestructiveInstType = DestructiveOther; 4203 let ElementSize = ElementSizeNone; 4204} 4205 4206multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 4207 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8>; 4208 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8>; 4209 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8>; 4210 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8>; 4211 4212 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 4213 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 4214 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 4215 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 4216} 4217 4218multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4219 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4220 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4221 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4222 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4223 4224 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4225 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4226 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4227 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4228} 4229 4230multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4231 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8>; 4232 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8>; 4233 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8>; 4234 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8>; 4235 4236 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _B)>; 4237 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _H)>; 4238 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat, !cast<Instruction>(NAME # _S)>; 4239 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat, !cast<Instruction>(NAME # _D)>; 4240} 4241 4242//===----------------------------------------------------------------------===// 4243// SVE Bitwise Logical - Unpredicated Group 4244//===----------------------------------------------------------------------===// 4245 4246class sve_int_bin_cons_log<bits<2> opc, string asm> 4247: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4248 asm, "\t$Zd, $Zn, $Zm", 4249 "", 4250 []>, Sched<[]> { 4251 bits<5> Zd; 4252 bits<5> Zm; 4253 bits<5> Zn; 4254 let Inst{31-24} = 0b00000100; 4255 let Inst{23-22} = opc{1-0}; 4256 let Inst{21} = 0b1; 4257 let Inst{20-16} = Zm; 4258 let Inst{15-10} = 0b001100; 4259 let Inst{9-5} = Zn; 4260 let Inst{4-0} = Zd; 4261} 4262 4263multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4264 def NAME : sve_int_bin_cons_log<opc, asm>; 4265 4266 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4267 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4268 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4269 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4270 4271 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4272 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4273 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4274 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4275 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4276 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4277} 4278 4279class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4280: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4281 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4282 "", 4283 []>, Sched<[]> { 4284 bits<5> Zdn; 4285 bits<5> Zk; 4286 bits<5> Zm; 4287 let Inst{31-24} = 0b00000100; 4288 let Inst{23-22} = opc{2-1}; 4289 let Inst{21} = 0b1; 4290 let Inst{20-16} = Zm; 4291 let Inst{15-11} = 0b00111; 4292 let Inst{10} = opc{0}; 4293 let Inst{9-5} = Zk; 4294 let Inst{4-0} = Zdn; 4295 4296 let Constraints = "$Zdn = $_Zdn"; 4297 let DestructiveInstType = DestructiveOther; 4298 let ElementSize = ElementSizeNone; 4299} 4300 4301multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op> { 4302 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4303 4304 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4305 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4306 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4307 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4308 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4309 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4310 4311 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4312 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4313 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4314 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4315} 4316 4317class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4318 ZPRRegOp zprty, Operand immtype> 4319: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4320 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4321 "", 4322 []>, Sched<[]> { 4323 bits<5> Zdn; 4324 bits<5> Zm; 4325 bits<6> imm; 4326 let Inst{31-24} = 0b00000100; 4327 let Inst{23-22} = tsz8_64{3-2}; 4328 let Inst{21} = 0b1; 4329 let Inst{20-19} = tsz8_64{1-0}; 4330 let Inst{18-16} = imm{2-0}; // imm3 4331 let Inst{15-10} = 0b001101; 4332 let Inst{9-5} = Zm; 4333 let Inst{4-0} = Zdn; 4334 4335 let Constraints = "$Zdn = $_Zdn"; 4336 let DestructiveInstType = DestructiveOther; 4337 let ElementSize = ElementSizeNone; 4338} 4339 4340multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4341 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4342 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4343 let Inst{19} = imm{3}; 4344 } 4345 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4346 let Inst{20-19} = imm{4-3}; 4347 } 4348 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4349 let Inst{22} = imm{5}; 4350 let Inst{20-19} = imm{4-3}; 4351 } 4352 4353 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4354 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4355 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4356 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4357} 4358 4359//===----------------------------------------------------------------------===// 4360// SVE Integer Wide Immediate - Predicated Group 4361//===----------------------------------------------------------------------===// 4362 4363class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4364 string asm, ZPRRegOp zprty> 4365: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4366 asm, "\t$Zd, $Pg/m, $imm8", 4367 "", 4368 []>, Sched<[]> { 4369 bits<4> Pg; 4370 bits<5> Zd; 4371 bits<8> imm8; 4372 let Inst{31-24} = 0b00000101; 4373 let Inst{23-22} = sz; 4374 let Inst{21-20} = 0b01; 4375 let Inst{19-16} = Pg; 4376 let Inst{15-13} = 0b110; 4377 let Inst{12-5} = imm8; 4378 let Inst{4-0} = Zd; 4379 4380 let Constraints = "$Zd = $_Zd"; 4381 let DestructiveInstType = DestructiveOther; 4382 let ElementSize = zprty.ElementSize; 4383} 4384 4385multiclass sve_int_dup_fpimm_pred<string asm> { 4386 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4387 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4388 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4389 4390 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4391 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4392 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4393 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4394 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4395 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4396} 4397 4398class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4399 ZPRRegOp zprty, string pred_qual, dag iops> 4400: I<(outs zprty:$Zd), iops, 4401 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4402 "", []>, Sched<[]> { 4403 bits<5> Zd; 4404 bits<4> Pg; 4405 bits<9> imm; 4406 let Inst{31-24} = 0b00000101; 4407 let Inst{23-22} = sz8_64; 4408 let Inst{21-20} = 0b01; 4409 let Inst{19-16} = Pg; 4410 let Inst{15} = 0b0; 4411 let Inst{14} = m; 4412 let Inst{13} = imm{8}; // sh 4413 let Inst{12-5} = imm{7-0}; // imm8 4414 let Inst{4-0} = Zd; 4415 4416 let DestructiveInstType = DestructiveOther; 4417 let ElementSize = zprty.ElementSize; 4418} 4419 4420multiclass sve_int_dup_imm_pred_merge_inst< 4421 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4422 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4423 let Constraints = "$Zd = $_Zd" in 4424 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4425 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4426 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4427 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4428 def : Pat<(intty 4429 (vselect predty:$Pg, 4430 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4431 intty:$Zd)), 4432 (!cast<Instruction>(NAME) zprty:$Zd, $Pg, i32:$imm, i32:$shift)>; 4433} 4434 4435multiclass sve_int_dup_imm_pred_merge<string asm> { 4436 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4437 i32, cpy_imm8_opt_lsl_i8>; 4438 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4439 i32, cpy_imm8_opt_lsl_i16>; 4440 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4441 i32, cpy_imm8_opt_lsl_i32>; 4442 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4443 i64, cpy_imm8_opt_lsl_i64>; 4444 4445 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4446 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4447 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4448 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4449 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4450 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4451} 4452 4453multiclass sve_int_dup_imm_pred_zero_inst< 4454 bits<2> sz8_64, string asm, ZPRRegOp zprty, ValueType intty, 4455 ValueType predty, ValueType scalarty, imm8_opt_lsl cpyimm> { 4456 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4457 (ins PPRAny:$Pg, cpyimm:$imm)>; 4458 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4459 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4460 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4461 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4462 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4463 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4464 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4465 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4466 def : Pat<(intty 4467 (vselect predty:$Pg, 4468 (intty (AArch64dup (scalarty (SVE8BitLslImm i32:$imm, i32:$shift)))), 4469 (intty (AArch64dup (scalarty 0))))), 4470 (!cast<Instruction>(NAME) $Pg, i32:$imm, i32:$shift)>; 4471} 4472 4473multiclass sve_int_dup_imm_pred_zero<string asm> { 4474 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, nxv16i8, nxv16i1, 4475 i32, cpy_imm8_opt_lsl_i8>; 4476 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, nxv8i16, nxv8i1, 4477 i32, cpy_imm8_opt_lsl_i16>; 4478 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, nxv4i32, nxv4i1, 4479 i32, cpy_imm8_opt_lsl_i32>; 4480 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, nxv2i64, nxv2i1, 4481 i64, cpy_imm8_opt_lsl_i64>; 4482} 4483 4484//===----------------------------------------------------------------------===// 4485// SVE Integer Compare - Vectors Group 4486//===----------------------------------------------------------------------===// 4487 4488class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4489 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4490: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4491 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4492 "", 4493 []>, Sched<[]> { 4494 bits<4> Pd; 4495 bits<3> Pg; 4496 bits<5> Zm; 4497 bits<5> Zn; 4498 let Inst{31-24} = 0b00100100; 4499 let Inst{23-22} = sz8_64; 4500 let Inst{21} = 0b0; 4501 let Inst{20-16} = Zm; 4502 let Inst{15} = opc{2}; 4503 let Inst{14} = cmp_1; 4504 let Inst{13} = opc{1}; 4505 let Inst{12-10} = Pg; 4506 let Inst{9-5} = Zn; 4507 let Inst{4} = opc{0}; 4508 let Inst{3-0} = Pd; 4509 4510 let Defs = [NZCV]; 4511 let ElementSize = pprty.ElementSize; 4512 let isPTestLike = 1; 4513} 4514 4515multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4516 ValueType intvt, Instruction cmp> { 4517 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4518 (cmp $Op1, $Op2, $Op3)>; 4519 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4520 (cmp $Op1, $Op3, $Op2)>; 4521} 4522 4523multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt, 4524 ValueType intvt, Instruction cmp> { 4525 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)), 4526 (cmp $Op1, $Op2)>; 4527 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)), 4528 (cmp $Op1, $Op2)>; 4529} 4530 4531multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4532 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4533 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4534 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4535 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4536 4537 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4538 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4539 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4540 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4541} 4542 4543multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4544 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4545 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4546 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4547 4548 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4549 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4550 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4551} 4552 4553multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4554 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4555 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4556 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4557 4558 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4559 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4560 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4561} 4562 4563 4564//===----------------------------------------------------------------------===// 4565// SVE Integer Compare - Signed Immediate Group 4566//===----------------------------------------------------------------------===// 4567 4568class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4569 ZPRRegOp zprty, 4570 Operand immtype> 4571: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4572 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4573 "", 4574 []>, Sched<[]> { 4575 bits<4> Pd; 4576 bits<3> Pg; 4577 bits<5> Zn; 4578 bits<5> imm5; 4579 let Inst{31-24} = 0b00100101; 4580 let Inst{23-22} = sz8_64; 4581 let Inst{21} = 0b0; 4582 let Inst{20-16} = imm5; 4583 let Inst{15} = opc{2}; 4584 let Inst{14} = 0b0; 4585 let Inst{13} = opc{1}; 4586 let Inst{12-10} = Pg; 4587 let Inst{9-5} = Zn; 4588 let Inst{4} = opc{0}; 4589 let Inst{3-0} = Pd; 4590 4591 let Defs = [NZCV]; 4592 let ElementSize = pprty.ElementSize; 4593 let isPTestLike = 1; 4594} 4595 4596multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4597 ValueType predvt, ValueType intvt, 4598 Operand immtype, Instruction cmp> { 4599 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4600 (intvt ZPR:$Zs1), 4601 (intvt (AArch64dup (immtype:$imm))), 4602 cc)), 4603 (cmp $Pg, $Zs1, immtype:$imm)>; 4604 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4605 (intvt (AArch64dup (immtype:$imm))), 4606 (intvt ZPR:$Zs1), 4607 commuted_cc)), 4608 (cmp $Pg, $Zs1, immtype:$imm)>; 4609} 4610 4611multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4612 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4613 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4614 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4615 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4616 4617 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4618 !cast<Instruction>(NAME # _B)>; 4619 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4620 !cast<Instruction>(NAME # _H)>; 4621 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4622 !cast<Instruction>(NAME # _S)>; 4623 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4624 !cast<Instruction>(NAME # _D)>; 4625} 4626 4627 4628//===----------------------------------------------------------------------===// 4629// SVE Integer Compare - Unsigned Immediate Group 4630//===----------------------------------------------------------------------===// 4631 4632class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4633 ZPRRegOp zprty, Operand immtype> 4634: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4635 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4636 "", 4637 []>, Sched<[]> { 4638 bits<4> Pd; 4639 bits<3> Pg; 4640 bits<5> Zn; 4641 bits<7> imm7; 4642 let Inst{31-24} = 0b00100100; 4643 let Inst{23-22} = sz8_64; 4644 let Inst{21} = 1; 4645 let Inst{20-14} = imm7; 4646 let Inst{13} = opc{1}; 4647 let Inst{12-10} = Pg; 4648 let Inst{9-5} = Zn; 4649 let Inst{4} = opc{0}; 4650 let Inst{3-0} = Pd; 4651 4652 let Defs = [NZCV]; 4653 let ElementSize = pprty.ElementSize; 4654 let isPTestLike = 1; 4655} 4656 4657multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4658 CondCode commuted_cc> { 4659 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4660 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4661 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4662 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4663 4664 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4665 !cast<Instruction>(NAME # _B)>; 4666 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4667 !cast<Instruction>(NAME # _H)>; 4668 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4669 !cast<Instruction>(NAME # _S)>; 4670 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 4671 !cast<Instruction>(NAME # _D)>; 4672} 4673 4674 4675//===----------------------------------------------------------------------===// 4676// SVE Integer Compare - Scalars Group 4677//===----------------------------------------------------------------------===// 4678 4679class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 4680: I<(outs), (ins rt:$Rn, rt:$Rm), 4681 asm, "\t$Rn, $Rm", 4682 "", 4683 []>, Sched<[]> { 4684 bits<5> Rm; 4685 bits<5> Rn; 4686 let Inst{31-23} = 0b001001011; 4687 let Inst{22} = sz; 4688 let Inst{21} = 0b1; 4689 let Inst{20-16} = Rm; 4690 let Inst{15-10} = 0b001000; 4691 let Inst{9-5} = Rn; 4692 let Inst{4} = opc; 4693 let Inst{3-0} = 0b0000; 4694 4695 let Defs = [NZCV]; 4696} 4697 4698class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 4699 RegisterClass gprty, PPRRegOp pprty> 4700: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 4701 asm, "\t$Pd, $Rn, $Rm", 4702 "", []>, Sched<[]> { 4703 bits<4> Pd; 4704 bits<5> Rm; 4705 bits<5> Rn; 4706 let Inst{31-24} = 0b00100101; 4707 let Inst{23-22} = sz8_64; 4708 let Inst{21} = 0b1; 4709 let Inst{20-16} = Rm; 4710 let Inst{15-13} = 0b000; 4711 let Inst{12-10} = opc{3-1}; 4712 let Inst{9-5} = Rn; 4713 let Inst{4} = opc{0}; 4714 let Inst{3-0} = Pd; 4715 4716 let Defs = [NZCV]; 4717 let ElementSize = pprty.ElementSize; 4718 let isWhile = 1; 4719} 4720 4721multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 4722 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 4723 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 4724 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 4725 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 4726 4727 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 4728 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 4729 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 4730 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 4731} 4732 4733multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 4734 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 4735 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 4736 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 4737 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 4738 4739 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 4740 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 4741 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 4742 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 4743} 4744 4745class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 4746 PPRRegOp pprty> 4747: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 4748 asm, "\t$Pd, $Rn, $Rm", 4749 "", []>, Sched<[]> { 4750 bits<4> Pd; 4751 bits<5> Rm; 4752 bits<5> Rn; 4753 let Inst{31-24} = 0b00100101; 4754 let Inst{23-22} = sz8_64; 4755 let Inst{21} = 0b1; 4756 let Inst{20-16} = Rm; 4757 let Inst{15-10} = 0b001100; 4758 let Inst{9-5} = Rn; 4759 let Inst{4} = rw; 4760 let Inst{3-0} = Pd; 4761 4762 let Defs = [NZCV]; 4763 let ElementSize = pprty.ElementSize; 4764 let isWhile = 1; 4765} 4766 4767multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 4768 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 4769 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 4770 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 4771 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 4772 4773 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 4774 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 4775 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 4776 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 4777} 4778 4779//===----------------------------------------------------------------------===// 4780// SVE Floating Point Fast Reduction Group 4781//===----------------------------------------------------------------------===// 4782 4783class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 4784 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4785: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 4786 asm, "\t$Vd, $Pg, $Zn", 4787 "", 4788 []>, Sched<[]> { 4789 bits<5> Zn; 4790 bits<5> Vd; 4791 bits<3> Pg; 4792 let Inst{31-24} = 0b01100101; 4793 let Inst{23-22} = sz; 4794 let Inst{21-19} = 0b000; 4795 let Inst{18-16} = opc; 4796 let Inst{15-13} = 0b001; 4797 let Inst{12-10} = Pg; 4798 let Inst{9-5} = Zn; 4799 let Inst{4-0} = Vd; 4800} 4801 4802multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 4803 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 4804 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 4805 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 4806 4807 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4808 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4809 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4810 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4811 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4812 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4813} 4814 4815//===----------------------------------------------------------------------===// 4816// SVE Floating Point Accumulating Reduction Group 4817//===----------------------------------------------------------------------===// 4818 4819class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 4820 ZPRRegOp zprty, FPRasZPROperand dstOpType> 4821: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 4822 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 4823 "", 4824 []>, 4825 Sched<[]> { 4826 bits<3> Pg; 4827 bits<5> Vdn; 4828 bits<5> Zm; 4829 let Inst{31-24} = 0b01100101; 4830 let Inst{23-22} = sz; 4831 let Inst{21-19} = 0b011; 4832 let Inst{18-16} = opc; 4833 let Inst{15-13} = 0b001; 4834 let Inst{12-10} = Pg; 4835 let Inst{9-5} = Zm; 4836 let Inst{4-0} = Vdn; 4837 4838 let Constraints = "$Vdn = $_Vdn"; 4839} 4840 4841multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 4842 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 4843 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 4844 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 4845 4846 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 4847 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 4848 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4849 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 4850 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4851 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4852} 4853 4854//===----------------------------------------------------------------------===// 4855// SVE Floating Point Compare - Vectors Group 4856//===----------------------------------------------------------------------===// 4857 4858class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4859 ZPRRegOp zprty> 4860: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 4861 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4862 "", 4863 []>, Sched<[]> { 4864 bits<4> Pd; 4865 bits<3> Pg; 4866 bits<5> Zm; 4867 bits<5> Zn; 4868 let Inst{31-24} = 0b01100101; 4869 let Inst{23-22} = sz; 4870 let Inst{21} = 0b0; 4871 let Inst{20-16} = Zm; 4872 let Inst{15} = opc{2}; 4873 let Inst{14} = 0b1; 4874 let Inst{13} = opc{1}; 4875 let Inst{12-10} = Pg; 4876 let Inst{9-5} = Zn; 4877 let Inst{4} = opc{0}; 4878 let Inst{3-0} = Pd; 4879} 4880 4881multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 4882 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4883 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4884 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4885 4886 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 4887 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 4888 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 4889} 4890 4891multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, 4892 CondCode cc1, CondCode cc2, 4893 CondCode invcc1, CondCode invcc2> { 4894 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4895 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4896 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4897 4898 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4899 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4900 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4901 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4902 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4903 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4904 4905 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4906 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4907 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4908 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4909 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4910 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4911} 4912 4913//===----------------------------------------------------------------------===// 4914// SVE Floating Point Compare - with Zero Group 4915//===----------------------------------------------------------------------===// 4916 4917class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 4918 ZPRRegOp zprty> 4919: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 4920 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 4921 "", 4922 []>, Sched<[]> { 4923 bits<4> Pd; 4924 bits<3> Pg; 4925 bits<5> Zn; 4926 let Inst{31-24} = 0b01100101; 4927 let Inst{23-22} = sz; 4928 let Inst{21-18} = 0b0100; 4929 let Inst{17-16} = opc{2-1}; 4930 let Inst{15-13} = 0b001; 4931 let Inst{12-10} = Pg; 4932 let Inst{9-5} = Zn; 4933 let Inst{4} = opc{0}; 4934 let Inst{3-0} = Pd; 4935} 4936 4937multiclass sve_fp_2op_p_pd<bits<3> opc, string asm, 4938 CondCode cc1, CondCode cc2, 4939 CondCode invcc1, CondCode invcc2> { 4940 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 4941 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 4942 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 4943 4944 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4945 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4946 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4947 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4948 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4949 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4950 4951 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4952 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4953 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4954 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4955 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4956 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4957} 4958 4959 4960//===----------------------------------------------------------------------===// 4961//SVE Index Generation Group 4962//===----------------------------------------------------------------------===// 4963 4964def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>; 4965def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>; 4966def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>; 4967def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>; 4968def i64imm_32bit_tgt : TImmLeaf<i64, [{ 4969 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); 4970}]>; 4971 4972class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 4973 Operand imm_ty> 4974: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 4975 asm, "\t$Zd, $imm5, $imm5b", 4976 "", []>, Sched<[]> { 4977 bits<5> Zd; 4978 bits<5> imm5; 4979 bits<5> imm5b; 4980 let Inst{31-24} = 0b00000100; 4981 let Inst{23-22} = sz8_64; 4982 let Inst{21} = 0b1; 4983 let Inst{20-16} = imm5b; 4984 let Inst{15-10} = 0b010000; 4985 let Inst{9-5} = imm5; 4986 let Inst{4-0} = Zd; 4987} 4988 4989multiclass sve_int_index_ii<string asm, SDPatternOperator step_vector, SDPatternOperator step_vector_oneuse> { 4990 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 4991 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 4992 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 4993 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 4994 4995 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)), 4996 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 4997 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)), 4998 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 4999 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)), 5000 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>; 5001 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)), 5002 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>; 5003 5004 // add(step_vector(step), dup(X)) -> index(X, step). 5005 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))), 5006 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5007 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))), 5008 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5009 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))), 5010 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 5011 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5012 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 5013} 5014 5015class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5016 RegisterClass srcRegType, Operand imm_ty> 5017: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 5018 asm, "\t$Zd, $imm5, $Rm", 5019 "", []>, Sched<[]> { 5020 bits<5> Rm; 5021 bits<5> Zd; 5022 bits<5> imm5; 5023 let Inst{31-24} = 0b00000100; 5024 let Inst{23-22} = sz8_64; 5025 let Inst{21} = 0b1; 5026 let Inst{20-16} = Rm; 5027 let Inst{15-10} = 0b010010; 5028 let Inst{9-5} = imm5; 5029 let Inst{4-0} = Zd; 5030} 5031 5032multiclass sve_int_index_ir<string asm, SDPatternOperator step_vector, SDPatternOperator step_vector_oneuse, SDPatternOperator mulop, SDPatternOperator muloneuseop> { 5033 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 5034 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 5035 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 5036 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 5037 5038 def : Pat<(nxv16i8 (step_vector i8:$imm)), 5039 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5040 def : Pat<(nxv8i16 (step_vector i16:$imm)), 5041 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5042 def : Pat<(nxv4i32 (step_vector i32:$imm)), 5043 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>; 5044 def : Pat<(nxv2i64 (step_vector i64:$imm)), 5045 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>; 5046 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)), 5047 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5048 5049 // add(step_vector(step), dup(X)) -> index(X, step). 5050 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))), 5051 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5052 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))), 5053 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5054 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))), 5055 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>; 5056 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5057 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>; 5058 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5059 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5060 5061 // mul(step_vector(1), dup(Y)) -> index(0, Y). 5062 def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), 5063 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>; 5064 def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), 5065 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>; 5066 def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), 5067 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>; 5068 def : Pat<(mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))), 5069 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>; 5070 5071 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5072 def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(simm5_8b:$imm5)))), 5073 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 5074 def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), (nxv8i16 (AArch64dup(simm5_16b:$imm5)))), 5075 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 5076 def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), (nxv4i32 (AArch64dup(simm5_32b:$imm5)))), 5077 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 5078 def : Pat<(add (muloneuseop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))), (nxv2i64 (AArch64dup(simm5_64b:$imm5)))), 5079 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 5080} 5081 5082class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5083 RegisterClass srcRegType, Operand imm_ty> 5084: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 5085 asm, "\t$Zd, $Rn, $imm5", 5086 "", []>, Sched<[]> { 5087 bits<5> Rn; 5088 bits<5> Zd; 5089 bits<5> imm5; 5090 let Inst{31-24} = 0b00000100; 5091 let Inst{23-22} = sz8_64; 5092 let Inst{21} = 0b1; 5093 let Inst{20-16} = imm5; 5094 let Inst{15-10} = 0b010001; 5095 let Inst{9-5} = Rn; 5096 let Inst{4-0} = Zd; 5097} 5098 5099multiclass sve_int_index_ri<string asm, SDPatternOperator step_vector, SDPatternOperator step_vector_oneuse> { 5100 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 5101 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 5102 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 5103 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 5104 5105 // add(step_vector(step), dup(X)) -> index(X, step). 5106 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), 5107 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5108 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))), 5109 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5110 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))), 5111 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 5112 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))), 5113 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 5114} 5115 5116class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5117 RegisterClass srcRegType> 5118: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 5119 asm, "\t$Zd, $Rn, $Rm", 5120 "", []>, Sched<[]> { 5121 bits<5> Zd; 5122 bits<5> Rm; 5123 bits<5> Rn; 5124 let Inst{31-24} = 0b00000100; 5125 let Inst{23-22} = sz8_64; 5126 let Inst{21} = 0b1; 5127 let Inst{20-16} = Rm; 5128 let Inst{15-10} = 0b010011; 5129 let Inst{9-5} = Rn; 5130 let Inst{4-0} = Zd; 5131} 5132 5133multiclass sve_int_index_rr<string asm, SDPatternOperator step_vector, SDPatternOperator step_vector_oneuse, SDPatternOperator mulop> { 5134 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 5135 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 5136 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 5137 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 5138 5139 // add(step_vector(step), dup(X)) -> index(X, step). 5140 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))), 5141 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5142 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (AArch64dup(i32 GPR32:$Rn)))), 5143 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5144 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (AArch64dup(i32 GPR32:$Rn)))), 5145 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>; 5146 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))), 5147 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>; 5148 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (AArch64dup(i64 GPR64:$Rn)))), 5149 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5150 5151 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5152 def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (AArch64dup(i32 GPR32:$Rm)))), (nxv16i8 (AArch64dup(i32 GPR32:$Rn)))), 5153 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>; 5154 def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (AArch64dup(i32 GPR32:$Rm)))),(nxv8i16 (AArch64dup(i32 GPR32:$Rn)))), 5155 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>; 5156 def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (AArch64dup(i32 GPR32:$Rm)))),(nxv4i32 (AArch64dup(i32 GPR32:$Rn)))), 5157 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>; 5158 def : Pat<(add (mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (AArch64dup(i64 GPR64:$Rm)))),(nxv2i64 (AArch64dup(i64 GPR64:$Rn)))), 5159 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>; 5160} 5161 5162//===----------------------------------------------------------------------===// 5163// SVE Bitwise Shift - Predicated Group 5164//===----------------------------------------------------------------------===// 5165 5166class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 5167 ZPRRegOp zprty, Operand immtype> 5168: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 5169 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 5170 "", 5171 []>, Sched<[]> { 5172 bits<3> Pg; 5173 bits<5> Zdn; 5174 bits<6> imm; 5175 let Inst{31-24} = 0b00000100; 5176 let Inst{23-22} = tsz8_64{3-2}; 5177 let Inst{21-20} = 0b00; 5178 let Inst{19-16} = opc; 5179 let Inst{15-13} = 0b100; 5180 let Inst{12-10} = Pg; 5181 let Inst{9-8} = tsz8_64{1-0}; 5182 let Inst{7-5} = imm{2-0}; // imm3 5183 let Inst{4-0} = Zdn; 5184 5185 let Constraints = "$Zdn = $_Zdn"; 5186 let DestructiveInstType = DestructiveBinaryImm; 5187 let ElementSize = zprty.ElementSize; 5188} 5189 5190multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 5191 SDPatternOperator op = null_frag> { 5192 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5193 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5194 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5195 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5196 let Inst{8} = imm{3}; 5197 } 5198 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5199 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5200 let Inst{9-8} = imm{4-3}; 5201 } 5202 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5203 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5204 let Inst{22} = imm{5}; 5205 let Inst{9-8} = imm{4-3}; 5206 } 5207 5208 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 5209 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 5210 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 5211 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 5212} 5213 5214// As above but shift amount takes the form of a "vector immediate". 5215multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 5216 string Ps, SDPatternOperator op> 5217: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 5218 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5219 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5220 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5221 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5222} 5223 5224multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 5225 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 5226 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 5227 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 5228 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 5229 5230 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 5231 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 5232 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 5233 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 5234} 5235 5236multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 5237 SDPatternOperator op = null_frag> { 5238 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5239 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5240 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5241 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5242 let Inst{8} = imm{3}; 5243 } 5244 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5245 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5246 let Inst{9-8} = imm{4-3}; 5247 } 5248 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5249 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5250 let Inst{22} = imm{5}; 5251 let Inst{9-8} = imm{4-3}; 5252 } 5253 5254 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 5255 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 5256 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 5257 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 5258} 5259 5260// As above but shift amount takes the form of a "vector immediate". 5261multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 5262 string Ps, SDPatternOperator op> 5263: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 5264 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5265 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5266 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5267 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5268} 5269 5270multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 5271 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 5272 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 5273 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 5274 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 5275 5276 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 5277 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 5278 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 5279 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 5280} 5281 5282class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 5283 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 5284: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 5285 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 5286 "", 5287 []>, Sched<[]> { 5288 bits<3> Pg; 5289 bits<5> Zdn; 5290 bits<5> Zm; 5291 let Inst{31-24} = 0b00000100; 5292 let Inst{23-22} = sz8_64; 5293 let Inst{21-20} = 0b01; 5294 let Inst{19} = wide; 5295 let Inst{18-16} = opc; 5296 let Inst{15-13} = 0b100; 5297 let Inst{12-10} = Pg; 5298 let Inst{9-5} = Zm; 5299 let Inst{4-0} = Zdn; 5300 5301 let Constraints = "$Zdn = $_Zdn"; 5302 let DestructiveInstType = DestructiveOther; 5303 let ElementSize = zprty.ElementSize; 5304} 5305 5306multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 5307 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 5308 let DestructiveInstType = DestructiveBinaryCommWithRev in { 5309 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 5310 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 5311 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 5312 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 5313 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 5314 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 5315 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 5316 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 5317 } 5318 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5319 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5320 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5321 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5322} 5323 5324multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5325 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5326 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5327 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5328 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5329 5330 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5331 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5332 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5333 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5334} 5335 5336multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5337 SDPatternOperator op> { 5338 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5339 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5340 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5341 5342 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5343 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5344 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5345} 5346 5347//===----------------------------------------------------------------------===// 5348// SVE Shift - Unpredicated Group 5349//===----------------------------------------------------------------------===// 5350 5351class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5352 ZPRRegOp zprty> 5353: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5354 asm, "\t$Zd, $Zn, $Zm", 5355 "", 5356 []>, Sched<[]> { 5357 bits<5> Zd; 5358 bits<5> Zm; 5359 bits<5> Zn; 5360 let Inst{31-24} = 0b00000100; 5361 let Inst{23-22} = sz8_64; 5362 let Inst{21} = 0b1; 5363 let Inst{20-16} = Zm; 5364 let Inst{15-12} = 0b1000; 5365 let Inst{11-10} = opc; 5366 let Inst{9-5} = Zn; 5367 let Inst{4-0} = Zd; 5368} 5369 5370multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> { 5371 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5372 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5373 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5374 5375 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5376 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5377 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5378} 5379 5380class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5381 ZPRRegOp zprty, Operand immtype> 5382: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5383 asm, "\t$Zd, $Zn, $imm", 5384 "", 5385 []>, Sched<[]> { 5386 bits<5> Zd; 5387 bits<5> Zn; 5388 bits<6> imm; 5389 let Inst{31-24} = 0b00000100; 5390 let Inst{23-22} = tsz8_64{3-2}; 5391 let Inst{21} = 0b1; 5392 let Inst{20-19} = tsz8_64{1-0}; 5393 let Inst{18-16} = imm{2-0}; // imm3 5394 let Inst{15-12} = 0b1001; 5395 let Inst{11-10} = opc; 5396 let Inst{9-5} = Zn; 5397 let Inst{4-0} = Zd; 5398} 5399 5400multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5401 SDPatternOperator op> { 5402 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5403 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5404 let Inst{19} = imm{3}; 5405 } 5406 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5407 let Inst{20-19} = imm{4-3}; 5408 } 5409 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5410 let Inst{22} = imm{5}; 5411 let Inst{20-19} = imm{4-3}; 5412 } 5413 5414 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5415 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5416 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5417 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5418} 5419 5420multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5421 SDPatternOperator op> { 5422 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5423 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5424 let Inst{19} = imm{3}; 5425 } 5426 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5427 let Inst{20-19} = imm{4-3}; 5428 } 5429 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5430 let Inst{22} = imm{5}; 5431 let Inst{20-19} = imm{4-3}; 5432 } 5433 5434 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5435 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5436 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5437 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5438} 5439 5440//===----------------------------------------------------------------------===// 5441// SVE Memory - Store Group 5442//===----------------------------------------------------------------------===// 5443 5444class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5445 RegisterOperand VecList> 5446: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5447 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5448 "", 5449 []>, Sched<[]> { 5450 bits<3> Pg; 5451 bits<5> Rn; 5452 bits<5> Zt; 5453 bits<4> imm4; 5454 let Inst{31-25} = 0b1110010; 5455 let Inst{24-23} = msz; 5456 let Inst{22-21} = esz; 5457 let Inst{20} = 0; 5458 let Inst{19-16} = imm4; 5459 let Inst{15-13} = 0b111; 5460 let Inst{12-10} = Pg; 5461 let Inst{9-5} = Rn; 5462 let Inst{4-0} = Zt; 5463 5464 let mayStore = 1; 5465} 5466 5467multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5468 RegisterOperand listty, ZPRRegOp zprty> 5469{ 5470 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5471 5472 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5473 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5474 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5475 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5476 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5477 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5478} 5479 5480class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5481 string asm, Operand immtype> 5482: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5483 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5484 "", 5485 []>, Sched<[]> { 5486 bits<3> Pg; 5487 bits<5> Rn; 5488 bits<5> Zt; 5489 bits<4> imm4; 5490 let Inst{31-25} = 0b1110010; 5491 let Inst{24-23} = sz; 5492 let Inst{22-21} = nregs; 5493 let Inst{20} = 1; 5494 let Inst{19-16} = imm4; 5495 let Inst{15-13} = 0b111; 5496 let Inst{12-10} = Pg; 5497 let Inst{9-5} = Rn; 5498 let Inst{4-0} = Zt; 5499 5500 let mayStore = 1; 5501} 5502 5503multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5504 string asm, Operand immtype> { 5505 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5506 5507 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5508 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5509} 5510 5511class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5512 string asm, RegisterOperand gprty> 5513: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5514 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5515 "", 5516 []>, Sched<[]> { 5517 bits<3> Pg; 5518 bits<5> Rm; 5519 bits<5> Rn; 5520 bits<5> Zt; 5521 let Inst{31-25} = 0b1110010; 5522 let Inst{24-23} = sz; 5523 let Inst{22-21} = nregs; 5524 let Inst{20-16} = Rm; 5525 let Inst{15-13} = 0b011; 5526 let Inst{12-10} = Pg; 5527 let Inst{9-5} = Rn; 5528 let Inst{4-0} = Zt; 5529 5530 let mayStore = 1; 5531} 5532 5533class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5534 RegisterOperand listty, RegisterOperand gprty> 5535: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5536 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5537 "", 5538 []>, Sched<[]> { 5539 bits<3> Pg; 5540 bits<5> Rm; 5541 bits<5> Rn; 5542 bits<5> Zt; 5543 let Inst{31-25} = 0b1110010; 5544 let Inst{24-21} = dtype; 5545 let Inst{20-16} = Rm; 5546 let Inst{15-13} = 0b010; 5547 let Inst{12-10} = Pg; 5548 let Inst{9-5} = Rn; 5549 let Inst{4-0} = Zt; 5550 5551 let mayStore = 1; 5552} 5553 5554multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5555 RegisterOperand listty, ZPRRegOp zprty, 5556 RegisterOperand gprty> { 5557 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5558 5559 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5560 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5561} 5562 5563class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5564: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5565 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5566 "", 5567 []>, Sched<[]> { 5568 bits<3> Pg; 5569 bits<5> Rn; 5570 bits<5> Zt; 5571 bits<4> imm4; 5572 let Inst{31-25} = 0b1110010; 5573 let Inst{24-23} = msz; 5574 let Inst{22-20} = 0b001; 5575 let Inst{19-16} = imm4; 5576 let Inst{15-13} = 0b111; 5577 let Inst{12-10} = Pg; 5578 let Inst{9-5} = Rn; 5579 let Inst{4-0} = Zt; 5580 5581 let mayStore = 1; 5582} 5583 5584multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5585 ZPRRegOp zprty> { 5586 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5587 5588 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5589 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5590 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5591 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5592 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5593 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5594} 5595 5596class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5597 RegisterOperand gprty> 5598: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5599 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5600 "", 5601 []>, Sched<[]> { 5602 bits<3> Pg; 5603 bits<5> Rm; 5604 bits<5> Rn; 5605 bits<5> Zt; 5606 let Inst{31-25} = 0b1110010; 5607 let Inst{24-23} = msz; 5608 let Inst{22-21} = 0b00; 5609 let Inst{20-16} = Rm; 5610 let Inst{15-13} = 0b011; 5611 let Inst{12-10} = Pg; 5612 let Inst{9-5} = Rn; 5613 let Inst{4-0} = Zt; 5614 5615 let mayStore = 1; 5616} 5617 5618multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 5619 ZPRRegOp zprty, RegisterOperand gprty> { 5620 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 5621 5622 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5623 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5624} 5625 5626class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 5627 RegisterOperand listty, ZPRRegOp zprty> 5628: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 5629 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 5630 "", 5631 []>, Sched<[]> { 5632 bits<3> Pg; 5633 bits<5> Rm; 5634 bits<5> Zn; 5635 bits<5> Zt; 5636 let Inst{31-25} = 0b1110010; 5637 let Inst{24-22} = opc; 5638 let Inst{21} = 0b0; 5639 let Inst{20-16} = Rm; 5640 let Inst{15-13} = 0b001; 5641 let Inst{12-10} = Pg; 5642 let Inst{9-5} = Zn; 5643 let Inst{4-0} = Zt; 5644 5645 let mayStore = 1; 5646} 5647 5648multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 5649 SDPatternOperator op, 5650 ValueType vt> { 5651 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 5652 5653 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5654 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 5655 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5656 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 5657 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5658 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 5659 5660 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 5661 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 5662} 5663 5664multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 5665 SDPatternOperator op, 5666 ValueType vt> { 5667 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 5668 5669 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 5670 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 5671 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5672 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 5673 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5674 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 5675 5676 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 5677 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 5678} 5679 5680class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 5681 RegisterOperand VecList, RegisterOperand zprext> 5682: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5683 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5684 "", 5685 []>, Sched<[]> { 5686 bits<3> Pg; 5687 bits<5> Rn; 5688 bits<5> Zm; 5689 bits<5> Zt; 5690 let Inst{31-25} = 0b1110010; 5691 let Inst{24-22} = opc; 5692 let Inst{21} = scaled; 5693 let Inst{20-16} = Zm; 5694 let Inst{15} = 0b1; 5695 let Inst{14} = xs; 5696 let Inst{13} = 0; 5697 let Inst{12-10} = Pg; 5698 let Inst{9-5} = Rn; 5699 let Inst{4-0} = Zt; 5700 5701 let mayStore = 1; 5702} 5703 5704multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 5705 SDPatternOperator sxtw_op, 5706 SDPatternOperator uxtw_op, 5707 RegisterOperand sxtw_opnd, 5708 RegisterOperand uxtw_opnd, 5709 ValueType vt > { 5710 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 5711 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 5712 5713 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5714 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5715 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5716 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5717 5718 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5719 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5720 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5721 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5722} 5723 5724multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 5725 SDPatternOperator sxtw_op, 5726 SDPatternOperator uxtw_op, 5727 RegisterOperand sxtw_opnd, 5728 RegisterOperand uxtw_opnd, 5729 ValueType vt > { 5730 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 5731 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 5732 5733 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5734 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5735 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5736 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5737 5738 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5739 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5740 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5741 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5742} 5743 5744multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 5745 SDPatternOperator sxtw_op, 5746 SDPatternOperator uxtw_op, 5747 RegisterOperand sxtw_opnd, 5748 RegisterOperand uxtw_opnd, 5749 ValueType vt> { 5750 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 5751 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 5752 5753 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5754 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5755 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5756 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5757 5758 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5759 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5760 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5761 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5762} 5763 5764multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 5765 SDPatternOperator sxtw_op, 5766 SDPatternOperator uxtw_op, 5767 RegisterOperand sxtw_opnd, 5768 RegisterOperand uxtw_opnd, 5769 ValueType vt> { 5770 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 5771 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 5772 5773 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5774 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 5775 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5776 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 5777 5778 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5779 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5780 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 5781 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5782} 5783 5784class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 5785 RegisterOperand zprext> 5786: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 5787 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 5788 "", 5789 []>, Sched<[]> { 5790 bits<3> Pg; 5791 bits<5> Rn; 5792 bits<5> Zm; 5793 bits<5> Zt; 5794 let Inst{31-25} = 0b1110010; 5795 let Inst{24-23} = msz; 5796 let Inst{22} = 0b0; 5797 let Inst{21} = scaled; 5798 let Inst{20-16} = Zm; 5799 let Inst{15-13} = 0b101; 5800 let Inst{12-10} = Pg; 5801 let Inst{9-5} = Rn; 5802 let Inst{4-0} = Zt; 5803 5804 let mayStore = 1; 5805} 5806 5807multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 5808 SDPatternOperator op, 5809 RegisterOperand zprext, 5810 ValueType vt> { 5811 def _SCALED_REAL : sve_mem_sst_sv2<msz, 1, asm, zprext>; 5812 5813 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5814 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 5815 5816 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 5817 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 5818} 5819 5820multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 5821 SDPatternOperator op, 5822 ValueType vt> { 5823 def _REAL : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 5824 5825 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 5826 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 5827 5828 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 5829 (!cast<Instruction>(NAME # _REAL) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 5830} 5831 5832class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 5833 RegisterOperand VecList, Operand imm_ty> 5834: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 5835 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 5836 "", 5837 []>, Sched<[]> { 5838 bits<3> Pg; 5839 bits<5> imm5; 5840 bits<5> Zn; 5841 bits<5> Zt; 5842 let Inst{31-25} = 0b1110010; 5843 let Inst{24-23} = opc{2-1}; 5844 let Inst{22} = 0b1; 5845 let Inst{21} = opc{0}; 5846 let Inst{20-16} = imm5; 5847 let Inst{15-13} = 0b101; 5848 let Inst{12-10} = Pg; 5849 let Inst{9-5} = Zn; 5850 let Inst{4-0} = Zt; 5851 5852 let mayStore = 1; 5853} 5854 5855multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 5856 Operand imm_ty, 5857 SDPatternOperator op, 5858 ValueType vt> { 5859 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 5860 5861 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5862 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 5863 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5864 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 5865 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5866 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 5867 5868 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 5869 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5870} 5871 5872multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 5873 Operand imm_ty, 5874 SDPatternOperator op, 5875 ValueType vt> { 5876 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 5877 5878 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5879 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 5880 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 5881 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 5882 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 5883 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 5884 5885 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 5886 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 5887} 5888 5889class sve_mem_z_spill<string asm> 5890: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 5891 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 5892 "", 5893 []>, Sched<[]> { 5894 bits<5> Rn; 5895 bits<5> Zt; 5896 bits<9> imm9; 5897 let Inst{31-22} = 0b1110010110; 5898 let Inst{21-16} = imm9{8-3}; 5899 let Inst{15-13} = 0b010; 5900 let Inst{12-10} = imm9{2-0}; 5901 let Inst{9-5} = Rn; 5902 let Inst{4-0} = Zt; 5903 5904 let mayStore = 1; 5905} 5906 5907multiclass sve_mem_z_spill<string asm> { 5908 def NAME : sve_mem_z_spill<asm>; 5909 5910 def : InstAlias<asm # "\t$Zt, [$Rn]", 5911 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 5912} 5913 5914class sve_mem_p_spill<string asm> 5915: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 5916 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 5917 "", 5918 []>, Sched<[]> { 5919 bits<4> Pt; 5920 bits<5> Rn; 5921 bits<9> imm9; 5922 let Inst{31-22} = 0b1110010110; 5923 let Inst{21-16} = imm9{8-3}; 5924 let Inst{15-13} = 0b000; 5925 let Inst{12-10} = imm9{2-0}; 5926 let Inst{9-5} = Rn; 5927 let Inst{4} = 0b0; 5928 let Inst{3-0} = Pt; 5929 5930 let mayStore = 1; 5931} 5932 5933multiclass sve_mem_p_spill<string asm> { 5934 def NAME : sve_mem_p_spill<asm>; 5935 5936 def : InstAlias<asm # "\t$Pt, [$Rn]", 5937 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 5938} 5939 5940//===----------------------------------------------------------------------===// 5941// SVE Permute - Predicates Group 5942//===----------------------------------------------------------------------===// 5943 5944class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 5945 PPRRegOp pprty> 5946: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 5947 asm, "\t$Pd, $Pn, $Pm", 5948 "", []>, Sched<[]> { 5949 bits<4> Pd; 5950 bits<4> Pm; 5951 bits<4> Pn; 5952 let Inst{31-24} = 0b00000101; 5953 let Inst{23-22} = sz8_64; 5954 let Inst{21-20} = 0b10; 5955 let Inst{19-16} = Pm; 5956 let Inst{15-13} = 0b010; 5957 let Inst{12-10} = opc; 5958 let Inst{9} = 0b0; 5959 let Inst{8-5} = Pn; 5960 let Inst{4} = 0b0; 5961 let Inst{3-0} = Pd; 5962} 5963 5964multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 5965 SDPatternOperator op> { 5966 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8>; 5967 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16>; 5968 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32>; 5969 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64>; 5970 5971 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 5972 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 5973 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 5974 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 5975} 5976 5977class sve_int_perm_punpk<bit opc, string asm> 5978: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 5979 asm, "\t$Pd, $Pn", 5980 "", 5981 []>, Sched<[]> { 5982 bits<4> Pd; 5983 bits<4> Pn; 5984 let Inst{31-17} = 0b000001010011000; 5985 let Inst{16} = opc; 5986 let Inst{15-9} = 0b0100000; 5987 let Inst{8-5} = Pn; 5988 let Inst{4} = 0b0; 5989 let Inst{3-0} = Pd; 5990} 5991 5992multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 5993 def NAME : sve_int_perm_punpk<opc, asm>; 5994 5995 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 5996 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 5997 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 5998} 5999 6000class sve_int_rdffr_pred<bit s, string asm> 6001: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 6002 asm, "\t$Pd, $Pg/z", 6003 "", 6004 []>, Sched<[]> { 6005 bits<4> Pd; 6006 bits<4> Pg; 6007 let Inst{31-23} = 0b001001010; 6008 let Inst{22} = s; 6009 let Inst{21-9} = 0b0110001111000; 6010 let Inst{8-5} = Pg; 6011 let Inst{4} = 0; 6012 let Inst{3-0} = Pd; 6013 6014 let Defs = !if(s, [NZCV], []); 6015 let Uses = [FFR]; 6016} 6017 6018multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 6019 def _REAL : sve_int_rdffr_pred<s, asm>; 6020 6021 // We need a layer of indirection because early machine code passes balk at 6022 // physical register (i.e. FFR) uses that have no previous definition. 6023 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6024 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 6025 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 6026 } 6027} 6028 6029class sve_int_rdffr_unpred<string asm> : I< 6030 (outs PPR8:$Pd), (ins), 6031 asm, "\t$Pd", 6032 "", 6033 []>, Sched<[]> { 6034 bits<4> Pd; 6035 let Inst{31-4} = 0b0010010100011001111100000000; 6036 let Inst{3-0} = Pd; 6037 6038 let Uses = [FFR]; 6039} 6040 6041multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 6042 def _REAL : sve_int_rdffr_unpred<asm>; 6043 6044 // We need a layer of indirection because early machine code passes balk at 6045 // physical register (i.e. FFR) uses that have no previous definition. 6046 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6047 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 6048 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 6049 } 6050} 6051 6052class sve_int_wrffr<string asm, SDPatternOperator op> 6053: I<(outs), (ins PPR8:$Pn), 6054 asm, "\t$Pn", 6055 "", 6056 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 6057 bits<4> Pn; 6058 let Inst{31-9} = 0b00100101001010001001000; 6059 let Inst{8-5} = Pn; 6060 let Inst{4-0} = 0b00000; 6061 6062 let hasSideEffects = 1; 6063 let Defs = [FFR]; 6064} 6065 6066class sve_int_setffr<string asm, SDPatternOperator op> 6067: I<(outs), (ins), 6068 asm, "", 6069 "", 6070 [(op)]>, Sched<[]> { 6071 let Inst{31-0} = 0b00100101001011001001000000000000; 6072 6073 let hasSideEffects = 1; 6074 let Defs = [FFR]; 6075} 6076 6077//===----------------------------------------------------------------------===// 6078// SVE Permute Vector - Predicated Group 6079//===----------------------------------------------------------------------===// 6080 6081class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 6082 ZPRRegOp zprty, RegisterClass rt> 6083: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 6084 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 6085 "", 6086 []>, Sched<[]> { 6087 bits<3> Pg; 6088 bits<5> Rdn; 6089 bits<5> Zm; 6090 let Inst{31-24} = 0b00000101; 6091 let Inst{23-22} = sz8_64; 6092 let Inst{21-17} = 0b11000; 6093 let Inst{16} = ab; 6094 let Inst{15-13} = 0b101; 6095 let Inst{12-10} = Pg; 6096 let Inst{9-5} = Zm; 6097 let Inst{4-0} = Rdn; 6098 6099 let Constraints = "$Rdn = $_Rdn"; 6100} 6101 6102multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 6103 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 6104 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 6105 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 6106 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 6107 6108 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 6109 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 6110 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6111 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6112} 6113 6114class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 6115 ZPRRegOp zprty, RegisterClass rt> 6116: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 6117 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 6118 "", 6119 []>, Sched<[]> { 6120 bits<3> Pg; 6121 bits<5> Vdn; 6122 bits<5> Zm; 6123 let Inst{31-24} = 0b00000101; 6124 let Inst{23-22} = sz8_64; 6125 let Inst{21-17} = 0b10101; 6126 let Inst{16} = ab; 6127 let Inst{15-13} = 0b100; 6128 let Inst{12-10} = Pg; 6129 let Inst{9-5} = Zm; 6130 let Inst{4-0} = Vdn; 6131 6132 let Constraints = "$Vdn = $_Vdn"; 6133} 6134 6135multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 6136 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 6137 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 6138 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 6139 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 6140 6141 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6142 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6143 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6144 6145 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6146} 6147 6148class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 6149 ZPRRegOp zprty> 6150: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6151 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6152 "", 6153 []>, Sched<[]> { 6154 bits<3> Pg; 6155 bits<5> Zdn; 6156 bits<5> Zm; 6157 let Inst{31-24} = 0b00000101; 6158 let Inst{23-22} = sz8_64; 6159 let Inst{21-17} = 0b10100; 6160 let Inst{16} = ab; 6161 let Inst{15-13} = 0b100; 6162 let Inst{12-10} = Pg; 6163 let Inst{9-5} = Zm; 6164 let Inst{4-0} = Zdn; 6165 6166 let Constraints = "$Zdn = $_Zdn"; 6167 let DestructiveInstType = DestructiveOther; 6168 let ElementSize = ElementSizeNone; 6169} 6170 6171multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 6172 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 6173 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 6174 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 6175 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 6176 6177 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6178 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6179 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6180 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6181 6182 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6183 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6184 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6185 6186 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6187} 6188 6189class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 6190 ZPRRegOp zprty, RegisterClass resultRegType> 6191: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 6192 asm, "\t$Rd, $Pg, $Zn", 6193 "", 6194 []>, Sched<[]> { 6195 bits<3> Pg; 6196 bits<5> Rd; 6197 bits<5> Zn; 6198 let Inst{31-24} = 0b00000101; 6199 let Inst{23-22} = sz8_64; 6200 let Inst{21-17} = 0b10000; 6201 let Inst{16} = ab; 6202 let Inst{15-13} = 0b101; 6203 let Inst{12-10} = Pg; 6204 let Inst{9-5} = Zn; 6205 let Inst{4-0} = Rd; 6206} 6207 6208multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 6209 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 6210 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 6211 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 6212 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 6213 6214 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6215 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6216 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6217 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6218} 6219 6220class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 6221 ZPRRegOp zprty, RegisterClass dstRegtype> 6222: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 6223 asm, "\t$Vd, $Pg, $Zn", 6224 "", 6225 []>, Sched<[]> { 6226 bits<3> Pg; 6227 bits<5> Vd; 6228 bits<5> Zn; 6229 let Inst{31-24} = 0b00000101; 6230 let Inst{23-22} = sz8_64; 6231 let Inst{21-17} = 0b10001; 6232 let Inst{16} = ab; 6233 let Inst{15-13} = 0b100; 6234 let Inst{12-10} = Pg; 6235 let Inst{9-5} = Zn; 6236 let Inst{4-0} = Vd; 6237} 6238 6239multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 6240 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 6241 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 6242 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 6243 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 6244 6245 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 6246 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6247 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 6248 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6249 6250 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6251} 6252 6253class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 6254: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6255 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6256 "", 6257 []>, Sched<[]> { 6258 bits<3> Pg; 6259 bits<5> Zdn; 6260 bits<5> Zm; 6261 let Inst{31-24} = 0b00000101; 6262 let Inst{23-22} = sz8_64; 6263 let Inst{21-13} = 0b101100100; 6264 let Inst{12-10} = Pg; 6265 let Inst{9-5} = Zm; 6266 let Inst{4-0} = Zdn; 6267 6268 let Constraints = "$Zdn = $_Zdn"; 6269 let DestructiveInstType = DestructiveOther; 6270 let ElementSize = ElementSizeNone; 6271} 6272 6273multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 6274 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 6275 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 6276 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 6277 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 6278 6279 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6280 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6281 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6282 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6283 6284 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6285 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6286 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6287 6288 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6289} 6290 6291class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 6292 ZPRRegOp zprty, RegisterOperand VecList> 6293: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 6294 asm, "\t$Zd, $Pg, $Zn", 6295 "", 6296 []>, Sched<[]> { 6297 bits<3> Pg; 6298 bits<5> Zn; 6299 bits<5> Zd; 6300 let Inst{31-24} = 0b00000101; 6301 let Inst{23-22} = sz8_64; 6302 let Inst{21-13} = 0b101101100; 6303 let Inst{12-10} = Pg; 6304 let Inst{9-5} = Zn; 6305 let Inst{4-0} = Zd; 6306} 6307 6308multiclass sve2_int_perm_splice_cons<string asm> { 6309 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 6310 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 6311 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 6312 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 6313} 6314 6315class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 6316 ZPRRegOp zprty> 6317: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 6318 asm, "\t$Zd, $Pg/m, $Zn", 6319 "", 6320 []>, Sched<[]> { 6321 bits<5> Zd; 6322 bits<3> Pg; 6323 bits<5> Zn; 6324 let Inst{31-24} = 0b00000101; 6325 let Inst{23-22} = sz8_64; 6326 let Inst{21-18} = 0b1001; 6327 let Inst{17-16} = opc; 6328 let Inst{15-13} = 0b100; 6329 let Inst{12-10} = Pg; 6330 let Inst{9-5} = Zn; 6331 let Inst{4-0} = Zd; 6332 6333 let Constraints = "$Zd = $_Zd"; 6334 let DestructiveInstType = DestructiveOther; 6335 let ElementSize = zprty.ElementSize; 6336} 6337 6338multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6339 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6340 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6341 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6342 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6343 6344 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6345 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6346 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6347 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6348} 6349 6350multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6351 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6352 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6353 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6354 6355 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6356 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6357 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6358} 6359 6360multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6361 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6362 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6363 6364 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6365 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6366} 6367 6368multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6369 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6370 6371 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6372} 6373 6374class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6375 RegisterClass srcRegType> 6376: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6377 asm, "\t$Zd, $Pg/m, $Rn", 6378 "", 6379 []>, Sched<[]> { 6380 bits<3> Pg; 6381 bits<5> Rn; 6382 bits<5> Zd; 6383 let Inst{31-24} = 0b00000101; 6384 let Inst{23-22} = sz8_64; 6385 let Inst{21-13} = 0b101000101; 6386 let Inst{12-10} = Pg; 6387 let Inst{9-5} = Rn; 6388 let Inst{4-0} = Zd; 6389 6390 let Constraints = "$Zd = $_Zd"; 6391 let DestructiveInstType = DestructiveOther; 6392 let ElementSize = zprty.ElementSize; 6393} 6394 6395multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6396 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6397 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6398 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6399 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6400 6401 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6402 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6403 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6404 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6405 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6406 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6407 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6408 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6409 6410 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6411 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6412 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6413 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6414 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6415 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6416 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6417 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6418} 6419 6420class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6421 RegisterClass srcRegtype> 6422: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6423 asm, "\t$Zd, $Pg/m, $Vn", 6424 "", 6425 []>, Sched<[]> { 6426 bits<3> Pg; 6427 bits<5> Vn; 6428 bits<5> Zd; 6429 let Inst{31-24} = 0b00000101; 6430 let Inst{23-22} = sz8_64; 6431 let Inst{21-13} = 0b100000100; 6432 let Inst{12-10} = Pg; 6433 let Inst{9-5} = Vn; 6434 let Inst{4-0} = Zd; 6435 6436 let Constraints = "$Zd = $_Zd"; 6437 let DestructiveInstType = DestructiveOther; 6438 let ElementSize = zprty.ElementSize; 6439} 6440 6441multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6442 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6443 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6444 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6445 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6446 6447 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6448 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6449 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6450 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6451 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6452 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6453 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6454 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6455 6456 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6457 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6458 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6459 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6460 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6461 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6462 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6463 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6464 6465 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6466 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6467} 6468 6469class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6470: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6471 asm, "\t$Zd, $Pg, $Zn", 6472 "", 6473 []>, Sched<[]> { 6474 bits<3> Pg; 6475 bits<5> Zd; 6476 bits<5> Zn; 6477 let Inst{31-23} = 0b000001011; 6478 let Inst{22} = sz; 6479 let Inst{21-13} = 0b100001100; 6480 let Inst{12-10} = Pg; 6481 let Inst{9-5} = Zn; 6482 let Inst{4-0} = Zd; 6483} 6484 6485multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6486 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6487 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6488 6489 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6490 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6491 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6492 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6493} 6494 6495//===----------------------------------------------------------------------===// 6496// SVE Memory - Contiguous Load Group 6497//===----------------------------------------------------------------------===// 6498 6499class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6500 RegisterOperand VecList> 6501: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6502 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6503 "", 6504 []>, Sched<[]> { 6505 bits<3> Pg; 6506 bits<5> Rn; 6507 bits<5> Zt; 6508 bits<4> imm4; 6509 let Inst{31-25} = 0b1010010; 6510 let Inst{24-21} = dtype; 6511 let Inst{20} = nf; 6512 let Inst{19-16} = imm4; 6513 let Inst{15-13} = 0b101; 6514 let Inst{12-10} = Pg; 6515 let Inst{9-5} = Rn; 6516 let Inst{4-0} = Zt; 6517 6518 let mayLoad = 1; 6519 let Uses = !if(nf, [FFR], []); 6520 let Defs = !if(nf, [FFR], []); 6521} 6522 6523multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6524 RegisterOperand listty, ZPRRegOp zprty> { 6525 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6526 6527 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6528 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6529 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6530 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6531 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6532 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6533 6534 // We need a layer of indirection because early machine code passes balk at 6535 // physical register (i.e. FFR) uses that have no previous definition. 6536 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6537 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6538 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6539 } 6540} 6541 6542multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6543 ZPRRegOp zprty> 6544: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6545 6546class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6547: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6548 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6549 "", 6550 []>, Sched<[]> { 6551 bits<5> Zt; 6552 bits<3> Pg; 6553 bits<5> Rn; 6554 bits<4> imm4; 6555 let Inst{31-25} = 0b1010010; 6556 let Inst{24-23} = msz; 6557 let Inst{22-20} = 0b000; 6558 let Inst{19-16} = imm4; 6559 let Inst{15-13} = 0b111; 6560 let Inst{12-10} = Pg; 6561 let Inst{9-5} = Rn; 6562 let Inst{4-0} = Zt; 6563 6564 let mayLoad = 1; 6565} 6566 6567multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6568 ZPRRegOp zprty> { 6569 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6570 6571 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6572 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6573 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6574 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6575 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6576 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6577} 6578 6579class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6580 RegisterOperand gprty> 6581: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6582 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6583 "", 6584 []>, Sched<[]> { 6585 bits<3> Pg; 6586 bits<5> Rm; 6587 bits<5> Rn; 6588 bits<5> Zt; 6589 let Inst{31-25} = 0b1010010; 6590 let Inst{24-23} = msz; 6591 let Inst{22-21} = 0b00; 6592 let Inst{20-16} = Rm; 6593 let Inst{15-13} = 0b110; 6594 let Inst{12-10} = Pg; 6595 let Inst{9-5} = Rn; 6596 let Inst{4-0} = Zt; 6597 6598 let mayLoad = 1; 6599} 6600 6601multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6602 ZPRRegOp zprty, RegisterOperand gprty> { 6603 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6604 6605 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6606 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6607} 6608 6609class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 6610: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 6611 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 6612 bits<5> Zt; 6613 bits<5> Rn; 6614 bits<3> Pg; 6615 bits<4> imm4; 6616 let Inst{31-25} = 0b1010010; 6617 let Inst{24-23} = sz; 6618 let Inst{22-20} = 0; 6619 let Inst{19-16} = imm4; 6620 let Inst{15-13} = 0b001; 6621 let Inst{12-10} = Pg; 6622 let Inst{9-5} = Rn; 6623 let Inst{4-0} = Zt; 6624 6625 let mayLoad = 1; 6626} 6627 6628multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 6629 ZPRRegOp zprty> { 6630 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 6631 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6632 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6633 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6634 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6635 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 6636 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 6637} 6638 6639class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 6640 RegisterOperand gprty> 6641: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6642 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 6643 bits<5> Zt; 6644 bits<3> Pg; 6645 bits<5> Rn; 6646 bits<5> Rm; 6647 let Inst{31-25} = 0b1010010; 6648 let Inst{24-23} = sz; 6649 let Inst{22-21} = 0; 6650 let Inst{20-16} = Rm; 6651 let Inst{15-13} = 0; 6652 let Inst{12-10} = Pg; 6653 let Inst{9-5} = Rn; 6654 let Inst{4-0} = Zt; 6655 6656 let mayLoad = 1; 6657} 6658 6659multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 6660 ZPRRegOp zprty, RegisterOperand gprty> { 6661 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 6662 6663 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6664 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6665} 6666 6667class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6668 RegisterOperand VecList, Operand immtype> 6669: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 6670 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 6671 "", 6672 []>, Sched<[]> { 6673 bits<3> Pg; 6674 bits<5> Rn; 6675 bits<5> Zt; 6676 bits<6> imm6; 6677 let Inst{31-25} = 0b1000010; 6678 let Inst{24-23} = dtypeh; 6679 let Inst{22} = 1; 6680 let Inst{21-16} = imm6; 6681 let Inst{15} = 0b1; 6682 let Inst{14-13} = dtypel; 6683 let Inst{12-10} = Pg; 6684 let Inst{9-5} = Rn; 6685 let Inst{4-0} = Zt; 6686 6687 let mayLoad = 1; 6688} 6689 6690multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 6691 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 6692 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 6693 6694 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6695 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6696 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 6697 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 6698 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6699 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6700} 6701 6702class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 6703 RegisterOperand VecList> 6704: I<(outs VecList:$Zt), iops, 6705 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6706 "", 6707 []>, Sched<[]> { 6708 bits<5> Zt; 6709 bits<3> Pg; 6710 bits<5> Rm; 6711 bits<5> Rn; 6712 let Inst{31-25} = 0b1010010; 6713 let Inst{24-21} = dtype; 6714 let Inst{20-16} = Rm; 6715 let Inst{15-14} = 0b01; 6716 let Inst{13} = ff; 6717 let Inst{12-10} = Pg; 6718 let Inst{9-5} = Rn; 6719 let Inst{4-0} = Zt; 6720 6721 let mayLoad = 1; 6722 let Uses = !if(ff, [FFR], []); 6723 let Defs = !if(ff, [FFR], []); 6724} 6725 6726multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 6727 ZPRRegOp zprty, RegisterOperand gprty> { 6728 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6729 asm, listty>; 6730 6731 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6732 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6733} 6734 6735multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 6736 ZPRRegOp zprty, RegisterOperand gprty> { 6737 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6738 asm, listty>; 6739 6740 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6741 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6742 6743 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6744 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 6745 6746 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6747 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 6748 6749 // We need a layer of indirection because early machine code passes balk at 6750 // physical register (i.e. FFR) uses that have no previous definition. 6751 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6752 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 6753 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 6754 } 6755} 6756 6757multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 6758 ZPRRegOp zprty> 6759: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 6760 6761class sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6762 string asm, Operand immtype> 6763: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 6764 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6765 "", 6766 []>, Sched<[]> { 6767 bits<5> Zt; 6768 bits<3> Pg; 6769 bits<5> Rn; 6770 bits<4> imm4; 6771 let Inst{31-25} = 0b1010010; 6772 let Inst{24-23} = sz; 6773 let Inst{22-21} = nregs; 6774 let Inst{20} = 0; 6775 let Inst{19-16} = imm4; 6776 let Inst{15-13} = 0b111; 6777 let Inst{12-10} = Pg; 6778 let Inst{9-5} = Rn; 6779 let Inst{4-0} = Zt; 6780 6781 let mayLoad = 1; 6782} 6783 6784multiclass sve_mem_eld_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6785 string asm, Operand immtype> { 6786 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 6787 6788 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6789 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6790} 6791 6792class sve_mem_eld_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 6793 string asm, RegisterOperand gprty> 6794: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6795 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6796 "", 6797 []>, Sched<[]> { 6798 bits<3> Pg; 6799 bits<5> Rm; 6800 bits<5> Rn; 6801 bits<5> Zt; 6802 let Inst{31-25} = 0b1010010; 6803 let Inst{24-23} = sz; 6804 let Inst{22-21} = nregs; 6805 let Inst{20-16} = Rm; 6806 let Inst{15-13} = 0b110; 6807 let Inst{12-10} = Pg; 6808 let Inst{9-5} = Rn; 6809 let Inst{4-0} = Zt; 6810 6811 let mayLoad = 1; 6812} 6813 6814//===----------------------------------------------------------------------===// 6815// SVE Memory - 32-bit Gather and Unsized Contiguous Group 6816//===----------------------------------------------------------------------===// 6817 6818// bit xs is '1' if offsets are signed 6819// bit scaled is '1' if the offsets are scaled 6820class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 6821 RegisterOperand zprext> 6822: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6823 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 6824 "", 6825 []>, Sched<[]> { 6826 bits<3> Pg; 6827 bits<5> Rn; 6828 bits<5> Zm; 6829 bits<5> Zt; 6830 let Inst{31-25} = 0b1000010; 6831 let Inst{24-23} = opc{3-2}; 6832 let Inst{22} = xs; 6833 let Inst{21} = scaled; 6834 let Inst{20-16} = Zm; 6835 let Inst{15} = 0b0; 6836 let Inst{14-13} = opc{1-0}; 6837 let Inst{12-10} = Pg; 6838 let Inst{9-5} = Rn; 6839 let Inst{4-0} = Zt; 6840 6841 let mayLoad = 1; 6842 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6843 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6844} 6845 6846multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 6847 SDPatternOperator sxtw_op, 6848 SDPatternOperator uxtw_op, 6849 RegisterOperand sxtw_opnd, 6850 RegisterOperand uxtw_opnd, 6851 ValueType vt> { 6852 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 6853 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 6854 6855 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6856 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6857 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6858 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6859 6860 // We need a layer of indirection because early machine code passes balk at 6861 // physical register (i.e. FFR) uses that have no previous definition. 6862 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6863 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6864 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6865 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6866 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6867 } 6868 6869 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6870 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6871 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 6872 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6873} 6874 6875multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 6876 SDPatternOperator sxtw_op, 6877 SDPatternOperator uxtw_op, 6878 RegisterOperand sxtw_opnd, 6879 RegisterOperand uxtw_opnd, 6880 ValueType vt> { 6881 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 6882 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 6883 6884 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6885 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6886 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 6887 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6888 6889 // We need a layer of indirection because early machine code passes balk at 6890 // physical register (i.e. FFR) uses that have no previous definition. 6891 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6892 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 6893 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 6894 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 6895 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 6896 } 6897 6898 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 6899 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6900 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 6901 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6902} 6903 6904 6905class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 6906: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 6907 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 6908 "", 6909 []>, Sched<[]> { 6910 bits<3> Pg; 6911 bits<5> Zn; 6912 bits<5> Zt; 6913 bits<5> imm5; 6914 let Inst{31-25} = 0b1000010; 6915 let Inst{24-23} = opc{3-2}; 6916 let Inst{22-21} = 0b01; 6917 let Inst{20-16} = imm5; 6918 let Inst{15} = 0b1; 6919 let Inst{14-13} = opc{1-0}; 6920 let Inst{12-10} = Pg; 6921 let Inst{9-5} = Zn; 6922 let Inst{4-0} = Zt; 6923 6924 let mayLoad = 1; 6925 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 6926 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 6927} 6928 6929multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 6930 SDPatternOperator op, ValueType vt> { 6931 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 6932 6933 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6934 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6935 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 6936 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6937 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 6938 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6939 6940 // We need a layer of indirection because early machine code passes balk at 6941 // physical register (i.e. FFR) uses that have no previous definition. 6942 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6943 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 6944 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 6945 } 6946 6947 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 6948 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6949} 6950 6951class sve_mem_prfm_si<bits<2> msz, string asm> 6952: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 6953 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 6954 "", 6955 []>, Sched<[]> { 6956 bits<5> Rn; 6957 bits<3> Pg; 6958 bits<6> imm6; 6959 bits<4> prfop; 6960 let Inst{31-22} = 0b1000010111; 6961 let Inst{21-16} = imm6; 6962 let Inst{15} = 0b0; 6963 let Inst{14-13} = msz; 6964 let Inst{12-10} = Pg; 6965 let Inst{9-5} = Rn; 6966 let Inst{4} = 0b0; 6967 let Inst{3-0} = prfop; 6968 6969 let hasSideEffects = 1; 6970} 6971 6972multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 6973 def NAME : sve_mem_prfm_si<msz, asm>; 6974 6975 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 6976 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6977} 6978 6979class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 6980: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6981 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 6982 "", 6983 []>, Sched<[]> { 6984 bits<5> Rm; 6985 bits<5> Rn; 6986 bits<3> Pg; 6987 bits<4> prfop; 6988 let Inst{31-25} = 0b1000010; 6989 let Inst{24-23} = opc{2-1}; 6990 let Inst{22-21} = 0b00; 6991 let Inst{20-16} = Rm; 6992 let Inst{15} = 0b1; 6993 let Inst{14} = opc{0}; 6994 let Inst{13} = 0b0; 6995 let Inst{12-10} = Pg; 6996 let Inst{9-5} = Rn; 6997 let Inst{4} = 0b0; 6998 let Inst{3-0} = prfop; 6999 7000 let hasSideEffects = 1; 7001} 7002 7003class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 7004 RegisterOperand zprext> 7005: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7006 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7007 "", 7008 []>, Sched<[]> { 7009 bits<3> Pg; 7010 bits<5> Rn; 7011 bits<5> Zm; 7012 bits<4> prfop; 7013 let Inst{31-23} = 0b100001000; 7014 let Inst{22} = xs; 7015 let Inst{21} = 0b1; 7016 let Inst{20-16} = Zm; 7017 let Inst{15} = 0b0; 7018 let Inst{14-13} = msz; 7019 let Inst{12-10} = Pg; 7020 let Inst{9-5} = Rn; 7021 let Inst{4} = 0b0; 7022 let Inst{3-0} = prfop; 7023 7024 let hasSideEffects = 1; 7025} 7026 7027multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 7028 RegisterOperand sxtw_opnd, 7029 RegisterOperand uxtw_opnd, 7030 SDPatternOperator op_sxtw, 7031 SDPatternOperator op_uxtw> { 7032 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 7033 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 7034 7035 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7036 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7037 7038 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7039 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7040} 7041 7042class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7043: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7044 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7045 "", 7046 []>, Sched<[]> { 7047 bits<3> Pg; 7048 bits<5> Zn; 7049 bits<5> imm5; 7050 bits<4> prfop; 7051 let Inst{31-25} = 0b1000010; 7052 let Inst{24-23} = msz; 7053 let Inst{22-21} = 0b00; 7054 let Inst{20-16} = imm5; 7055 let Inst{15-13} = 0b111; 7056 let Inst{12-10} = Pg; 7057 let Inst{9-5} = Zn; 7058 let Inst{4} = 0b0; 7059 let Inst{3-0} = prfop; 7060} 7061 7062multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7063 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 7064 7065 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7066 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7067 7068 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7069 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7070} 7071 7072class sve_mem_z_fill<string asm> 7073: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 7074 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 7075 "", 7076 []>, Sched<[]> { 7077 bits<5> Rn; 7078 bits<5> Zt; 7079 bits<9> imm9; 7080 let Inst{31-22} = 0b1000010110; 7081 let Inst{21-16} = imm9{8-3}; 7082 let Inst{15-13} = 0b010; 7083 let Inst{12-10} = imm9{2-0}; 7084 let Inst{9-5} = Rn; 7085 let Inst{4-0} = Zt; 7086 7087 let mayLoad = 1; 7088} 7089 7090multiclass sve_mem_z_fill<string asm> { 7091 def NAME : sve_mem_z_fill<asm>; 7092 7093 def : InstAlias<asm # "\t$Zt, [$Rn]", 7094 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 7095} 7096 7097class sve_mem_p_fill<string asm> 7098: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 7099 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 7100 "", 7101 []>, Sched<[]> { 7102 bits<4> Pt; 7103 bits<5> Rn; 7104 bits<9> imm9; 7105 let Inst{31-22} = 0b1000010110; 7106 let Inst{21-16} = imm9{8-3}; 7107 let Inst{15-13} = 0b000; 7108 let Inst{12-10} = imm9{2-0}; 7109 let Inst{9-5} = Rn; 7110 let Inst{4} = 0b0; 7111 let Inst{3-0} = Pt; 7112 7113 let mayLoad = 1; 7114} 7115 7116multiclass sve_mem_p_fill<string asm> { 7117 def NAME : sve_mem_p_fill<asm>; 7118 7119 def : InstAlias<asm # "\t$Pt, [$Rn]", 7120 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 7121} 7122 7123class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 7124 RegisterOperand VecList> 7125: I<(outs VecList:$Zt), iops, 7126 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 7127 "", 7128 []>, Sched<[]> { 7129 bits<3> Pg; 7130 bits<5> Rm; 7131 bits<5> Zn; 7132 bits<5> Zt; 7133 let Inst{31} = 0b1; 7134 let Inst{30} = opc{4}; 7135 let Inst{29-25} = 0b00010; 7136 let Inst{24-23} = opc{3-2}; 7137 let Inst{22-21} = 0b00; 7138 let Inst{20-16} = Rm; 7139 let Inst{15} = 0b1; 7140 let Inst{14-13} = opc{1-0}; 7141 let Inst{12-10} = Pg; 7142 let Inst{9-5} = Zn; 7143 let Inst{4-0} = Zt; 7144 7145 let mayLoad = 1; 7146} 7147 7148multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 7149 SDPatternOperator op, 7150 ValueType vt> { 7151 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 7152 asm, Z_s>; 7153 7154 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7155 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 7156 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7157 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 7158 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7159 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 7160 7161 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 7162 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 7163} 7164 7165multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 7166 SDPatternOperator op, 7167 ValueType vt> { 7168 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 7169 asm, Z_d>; 7170 7171 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7172 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 7173 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7174 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 7175 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7176 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 7177 7178 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 7179 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 7180} 7181 7182//===----------------------------------------------------------------------===// 7183// SVE Memory - 64-bit Gather Group 7184//===----------------------------------------------------------------------===// 7185 7186// bit xs is '1' if offsets are signed 7187// bit scaled is '1' if the offsets are scaled 7188// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7189class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 7190 RegisterOperand zprext> 7191: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7192 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7193 "", 7194 []>, Sched<[]> { 7195 bits<3> Pg; 7196 bits<5> Rn; 7197 bits<5> Zm; 7198 bits<5> Zt; 7199 let Inst{31-25} = 0b1100010; 7200 let Inst{24-23} = opc{3-2}; 7201 let Inst{22} = xs; 7202 let Inst{21} = scaled; 7203 let Inst{20-16} = Zm; 7204 let Inst{15} = lsl; 7205 let Inst{14-13} = opc{1-0}; 7206 let Inst{12-10} = Pg; 7207 let Inst{9-5} = Rn; 7208 let Inst{4-0} = Zt; 7209 7210 let mayLoad = 1; 7211 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7212 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7213} 7214 7215multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 7216 SDPatternOperator sxtw_op, 7217 SDPatternOperator uxtw_op, 7218 RegisterOperand sxtw_opnd, 7219 RegisterOperand uxtw_opnd, 7220 ValueType vt> { 7221 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 7222 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 7223 7224 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7225 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7226 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7227 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7228 7229 // We need a layer of indirection because early machine code passes balk at 7230 // physical register (i.e. FFR) uses that have no previous definition. 7231 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7232 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7233 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7234 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7235 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7236 } 7237 7238 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7239 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7240 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7241 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7242} 7243 7244multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 7245 SDPatternOperator sxtw_op, 7246 SDPatternOperator uxtw_op, 7247 RegisterOperand sxtw_opnd, 7248 RegisterOperand uxtw_opnd, 7249 ValueType vt> { 7250 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 7251 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 7252 7253 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7254 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7255 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7256 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7257 7258 // We need a layer of indirection because early machine code passes balk at 7259 // physical register (i.e. FFR) uses that have no previous definition. 7260 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7261 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7262 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7263 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7264 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7265 } 7266 7267 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7268 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7269 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7270 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7271} 7272 7273multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 7274 SDPatternOperator op, 7275 RegisterOperand zprext, ValueType vt> { 7276 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 7277 7278 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7279 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 7280 7281 // We need a layer of indirection because early machine code passes balk at 7282 // physical register (i.e. FFR) uses that have no previous definition. 7283 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7284 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 7285 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7286 } 7287 7288 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7289 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7290} 7291 7292multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 7293 SDPatternOperator op, ValueType vt> { 7294 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 7295 7296 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7297 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 7298 7299 // We need a layer of indirection because early machine code passes balk at 7300 // physical register (i.e. FFR) uses that have no previous definition. 7301 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7302 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 7303 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 7304 } 7305 7306 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7307 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7308} 7309 7310class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7311: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7312 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7313 "", 7314 []>, Sched<[]> { 7315 bits<3> Pg; 7316 bits<5> Zn; 7317 bits<5> Zt; 7318 bits<5> imm5; 7319 let Inst{31-25} = 0b1100010; 7320 let Inst{24-23} = opc{3-2}; 7321 let Inst{22-21} = 0b01; 7322 let Inst{20-16} = imm5; 7323 let Inst{15} = 0b1; 7324 let Inst{14-13} = opc{1-0}; 7325 let Inst{12-10} = Pg; 7326 let Inst{9-5} = Zn; 7327 let Inst{4-0} = Zt; 7328 7329 let mayLoad = 1; 7330 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7331 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7332} 7333 7334multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7335 SDPatternOperator op, ValueType vt> { 7336 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7337 7338 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7339 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7340 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7341 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7342 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7343 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7344 7345 // We need a layer of indirection because early machine code passes balk at 7346 // physical register (i.e. FFR) uses that have no previous definition. 7347 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7348 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7349 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7350 } 7351 7352 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7353 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7354} 7355 7356// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7357class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7358 RegisterOperand zprext> 7359: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7360 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7361 "", 7362 []>, Sched<[]> { 7363 bits<3> Pg; 7364 bits<5> Rn; 7365 bits<5> Zm; 7366 bits<4> prfop; 7367 let Inst{31-23} = 0b110001000; 7368 let Inst{22} = xs; 7369 let Inst{21} = 0b1; 7370 let Inst{20-16} = Zm; 7371 let Inst{15} = lsl; 7372 let Inst{14-13} = msz; 7373 let Inst{12-10} = Pg; 7374 let Inst{9-5} = Rn; 7375 let Inst{4} = 0b0; 7376 let Inst{3-0} = prfop; 7377 7378 let hasSideEffects = 1; 7379} 7380 7381multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7382 RegisterOperand sxtw_opnd, 7383 RegisterOperand uxtw_opnd, 7384 SDPatternOperator op_sxtw, 7385 SDPatternOperator op_uxtw> { 7386 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7387 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7388 7389 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7390 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7391 7392 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7393 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7394 7395} 7396 7397multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7398 RegisterOperand zprext, SDPatternOperator frag> { 7399 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7400 7401 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7402 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7403 7404} 7405 7406class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7407: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7408 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7409 "", 7410 []>, Sched<[]> { 7411 bits<3> Pg; 7412 bits<5> Zn; 7413 bits<5> imm5; 7414 bits<4> prfop; 7415 let Inst{31-25} = 0b1100010; 7416 let Inst{24-23} = msz; 7417 let Inst{22-21} = 0b00; 7418 let Inst{20-16} = imm5; 7419 let Inst{15-13} = 0b111; 7420 let Inst{12-10} = Pg; 7421 let Inst{9-5} = Zn; 7422 let Inst{4} = 0b0; 7423 let Inst{3-0} = prfop; 7424 7425 let hasSideEffects = 1; 7426} 7427 7428multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7429 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7430 7431 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7432 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7433 7434 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7435 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7436} 7437 7438//===----------------------------------------------------------------------===// 7439// SVE Compute Vector Address Group 7440//===----------------------------------------------------------------------===// 7441 7442class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7443 ZPRRegOp zprty, RegisterOperand zprext> 7444: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7445 asm, "\t$Zd, [$Zn, $Zm]", 7446 "", 7447 []>, Sched<[]> { 7448 bits<5> Zd; 7449 bits<5> Zn; 7450 bits<5> Zm; 7451 let Inst{31-24} = 0b00000100; 7452 let Inst{23-22} = opc; 7453 let Inst{21} = 0b1; 7454 let Inst{20-16} = Zm; 7455 let Inst{15-12} = 0b1010; 7456 let Inst{11-10} = msz; 7457 let Inst{9-5} = Zn; 7458 let Inst{4-0} = Zd; 7459} 7460 7461multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7462 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7463 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7464 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7465 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7466} 7467 7468multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7469 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7470 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7471 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7472 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7473} 7474 7475multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7476 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7477 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7478 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7479 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7480} 7481 7482multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7483 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7484 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7485 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7486 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7487} 7488 7489//===----------------------------------------------------------------------===// 7490// SVE Integer Misc - Unpredicated Group 7491//===----------------------------------------------------------------------===// 7492 7493class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7494: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7495 asm, "\t$Zd, $Zn, $Zm", 7496 "", 7497 []>, Sched<[]> { 7498 bits<5> Zd; 7499 bits<5> Zm; 7500 bits<5> Zn; 7501 let Inst{31-24} = 0b00000100; 7502 let Inst{23-22} = sz; 7503 let Inst{21} = 0b1; 7504 let Inst{20-16} = Zm; 7505 let Inst{15-10} = 0b101100; 7506 let Inst{9-5} = Zn; 7507 let Inst{4-0} = Zd; 7508} 7509 7510multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7511 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7512 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7513 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7514 7515 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7516 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7517 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7518} 7519 7520class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7521: I<(outs zprty:$Zd), (ins zprty:$Zn), 7522 asm, "\t$Zd, $Zn", 7523 "", 7524 []>, Sched<[]> { 7525 bits<5> Zd; 7526 bits<5> Zn; 7527 let Inst{31-24} = 0b00000100; 7528 let Inst{23-22} = opc{7-6}; 7529 let Inst{21} = 0b1; 7530 let Inst{20-16} = opc{5-1}; 7531 let Inst{15-11} = 0b10111; 7532 let Inst{10} = opc{0}; 7533 let Inst{9-5} = Zn; 7534 let Inst{4-0} = Zd; 7535} 7536 7537multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7538 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7539 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7540 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7541 7542 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7543 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7544 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7545} 7546 7547//===----------------------------------------------------------------------===// 7548// SVE Integer Reduction Group 7549//===----------------------------------------------------------------------===// 7550 7551class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7552 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7553: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7554 asm, "\t$Vd, $Pg, $Zn", 7555 "", 7556 []>, Sched<[]> { 7557 bits<3> Pg; 7558 bits<5> Vd; 7559 bits<5> Zn; 7560 let Inst{31-24} = 0b00000100; 7561 let Inst{23-22} = sz8_32; 7562 let Inst{21} = 0b0; 7563 let Inst{20-19} = fmt; 7564 let Inst{18-16} = opc; 7565 let Inst{15-13} = 0b001; 7566 let Inst{12-10} = Pg; 7567 let Inst{9-5} = Zn; 7568 let Inst{4-0} = Vd; 7569} 7570 7571multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7572 SDPatternOperator op> { 7573 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7574 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7575 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7576 7577 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7578 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7579 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7580} 7581 7582multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7583 SDPatternOperator op> { 7584 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7585 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7586 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7587 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7588 7589 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7590 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7591 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7592 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7593} 7594 7595multiclass sve_int_reduce_1<bits<3> opc, string asm, 7596 SDPatternOperator op> { 7597 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7598 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7599 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7600 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7601 7602 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7603 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7604 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7605 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7606} 7607 7608multiclass sve_int_reduce_2<bits<3> opc, string asm, 7609 SDPatternOperator op> { 7610 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 7611 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 7612 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 7613 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 7614 7615 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7616 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7617 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7618 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7619} 7620 7621class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 7622 ZPRRegOp zprty, string pg_suffix, dag iops> 7623: I<(outs zprty:$Zd), iops, 7624 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 7625 "", 7626 []>, Sched<[]> { 7627 bits<3> Pg; 7628 bits<5> Zd; 7629 bits<5> Zn; 7630 let Inst{31-24} = 0b00000100; 7631 let Inst{23-22} = sz8_32; 7632 let Inst{21-19} = 0b010; 7633 let Inst{18-16} = opc; 7634 let Inst{15-13} = 0b001; 7635 let Inst{12-10} = Pg; 7636 let Inst{9-5} = Zn; 7637 let Inst{4-0} = Zd; 7638 7639 let ElementSize = zprty.ElementSize; 7640} 7641 7642multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 7643let Constraints = "$Zd = $_Zd" in { 7644 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 7645 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 7646 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 7647 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 7648 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 7649 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 7650 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 7651 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 7652} 7653} 7654 7655multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 7656 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 7657 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 7658 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 7659 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 7660 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 7661 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 7662 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 7663 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 7664} 7665 7666//===----------------------------------------------------------------------===// 7667// SVE Propagate Break Group 7668//===----------------------------------------------------------------------===// 7669 7670class sve_int_brkp<bits<2> opc, string asm> 7671: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 7672 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 7673 "", 7674 []>, Sched<[]> { 7675 bits<4> Pd; 7676 bits<4> Pg; 7677 bits<4> Pm; 7678 bits<4> Pn; 7679 let Inst{31-24} = 0b00100101; 7680 let Inst{23} = 0b0; 7681 let Inst{22} = opc{1}; 7682 let Inst{21-20} = 0b00; 7683 let Inst{19-16} = Pm; 7684 let Inst{15-14} = 0b11; 7685 let Inst{13-10} = Pg; 7686 let Inst{9} = 0b0; 7687 let Inst{8-5} = Pn; 7688 let Inst{4} = opc{0}; 7689 let Inst{3-0} = Pd; 7690 7691 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7692} 7693 7694multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 7695 def NAME : sve_int_brkp<opc, asm>; 7696 7697 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7698} 7699 7700 7701//===----------------------------------------------------------------------===// 7702// SVE Partition Break Group 7703//===----------------------------------------------------------------------===// 7704 7705class sve_int_brkn<bit S, string asm> 7706: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 7707 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 7708 "", 7709 []>, Sched<[]> { 7710 bits<4> Pdm; 7711 bits<4> Pg; 7712 bits<4> Pn; 7713 let Inst{31-23} = 0b001001010; 7714 let Inst{22} = S; 7715 let Inst{21-14} = 0b01100001; 7716 let Inst{13-10} = Pg; 7717 let Inst{9} = 0b0; 7718 let Inst{8-5} = Pn; 7719 let Inst{4} = 0b0; 7720 let Inst{3-0} = Pdm; 7721 7722 let Constraints = "$Pdm = $_Pdm"; 7723 let Defs = !if(S, [NZCV], []); 7724} 7725 7726multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 7727 def NAME : sve_int_brkn<opc, asm>; 7728 7729 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7730} 7731 7732class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 7733: I<(outs PPR8:$Pd), iops, 7734 asm, "\t$Pd, $Pg"#suffix#", $Pn", 7735 "", 7736 []>, Sched<[]> { 7737 bits<4> Pd; 7738 bits<4> Pg; 7739 bits<4> Pn; 7740 let Inst{31-24} = 0b00100101; 7741 let Inst{23-22} = opc{2-1}; 7742 let Inst{21-14} = 0b01000001; 7743 let Inst{13-10} = Pg; 7744 let Inst{9} = 0b0; 7745 let Inst{8-5} = Pn; 7746 let Inst{4} = opc{0}; 7747 let Inst{3-0} = Pd; 7748 7749 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 7750 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 7751 7752} 7753 7754multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 7755 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 7756 7757 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7758} 7759 7760multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 7761 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 7762 7763 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 7764} 7765 7766//===----------------------------------------------------------------------===// 7767// SVE2 String Processing Group 7768//===----------------------------------------------------------------------===// 7769 7770class sve2_char_match<bit sz, bit opc, string asm, 7771 PPRRegOp pprty, ZPRRegOp zprty> 7772: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7773 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 7774 "", 7775 []>, Sched<[]> { 7776 bits<4> Pd; 7777 bits<3> Pg; 7778 bits<5> Zm; 7779 bits<5> Zn; 7780 let Inst{31-23} = 0b010001010; 7781 let Inst{22} = sz; 7782 let Inst{21} = 0b1; 7783 let Inst{20-16} = Zm; 7784 let Inst{15-13} = 0b100; 7785 let Inst{12-10} = Pg; 7786 let Inst{9-5} = Zn; 7787 let Inst{4} = opc; 7788 let Inst{3-0} = Pd; 7789 7790 let Defs = [NZCV]; 7791 let isPTestLike = 1; 7792} 7793 7794multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 7795 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 7796 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 7797 7798 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 7799 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7800} 7801 7802//===----------------------------------------------------------------------===// 7803// SVE2 Histogram Computation - Segment Group 7804//===----------------------------------------------------------------------===// 7805 7806class sve2_hist_gen_segment<string asm, SDPatternOperator op> 7807: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 7808 asm, "\t$Zd, $Zn, $Zm", 7809 "", 7810 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 7811 bits<5> Zd; 7812 bits<5> Zn; 7813 bits<5> Zm; 7814 let Inst{31-21} = 0b01000101001; 7815 let Inst{20-16} = Zm; 7816 let Inst{15-10} = 0b101000; 7817 let Inst{9-5} = Zn; 7818 let Inst{4-0} = Zd; 7819} 7820 7821//===----------------------------------------------------------------------===// 7822// SVE2 Histogram Computation - Vector Group 7823//===----------------------------------------------------------------------===// 7824 7825class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 7826: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 7827 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 7828 "", 7829 []>, Sched<[]> { 7830 bits<5> Zd; 7831 bits<5> Zn; 7832 bits<3> Pg; 7833 bits<5> Zm; 7834 let Inst{31-23} = 0b010001011; 7835 let Inst{22} = sz; 7836 let Inst{21} = 0b1; 7837 let Inst{20-16} = Zm; 7838 let Inst{15-13} = 0b110; 7839 let Inst{12-10} = Pg; 7840 let Inst{9-5} = Zn; 7841 let Inst{4-0} = Zd; 7842} 7843 7844multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 7845 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 7846 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 7847 7848 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7849 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7850} 7851 7852//===----------------------------------------------------------------------===// 7853// SVE2 Crypto Extensions Group 7854//===----------------------------------------------------------------------===// 7855 7856class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 7857: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7858 asm, "\t$Zd, $Zn, $Zm", 7859 "", 7860 []>, Sched<[]> { 7861 bits<5> Zd; 7862 bits<5> Zn; 7863 bits<5> Zm; 7864 let Inst{31-21} = 0b01000101001; 7865 let Inst{20-16} = Zm; 7866 let Inst{15-11} = 0b11110; 7867 let Inst{10} = opc; 7868 let Inst{9-5} = Zn; 7869 let Inst{4-0} = Zd; 7870} 7871 7872multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 7873 SDPatternOperator op, ValueType vt> { 7874 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 7875 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7876} 7877 7878class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 7879: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 7880 asm, "\t$Zdn, $_Zdn, $Zm", 7881 "", 7882 []>, Sched<[]> { 7883 bits<5> Zdn; 7884 bits<5> Zm; 7885 let Inst{31-17} = 0b010001010010001; 7886 let Inst{16} = opc{1}; 7887 let Inst{15-11} = 0b11100; 7888 let Inst{10} = opc{0}; 7889 let Inst{9-5} = Zm; 7890 let Inst{4-0} = Zdn; 7891 7892 let Constraints = "$Zdn = $_Zdn"; 7893} 7894 7895multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 7896 SDPatternOperator op, ValueType vt> { 7897 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 7898 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 7899} 7900 7901class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 7902: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 7903 asm, "\t$Zdn, $_Zdn", 7904 "", 7905 []>, Sched<[]> { 7906 bits<5> Zdn; 7907 let Inst{31-11} = 0b010001010010000011100; 7908 let Inst{10} = opc; 7909 let Inst{9-5} = 0b00000; 7910 let Inst{4-0} = Zdn; 7911 7912 let Constraints = "$Zdn = $_Zdn"; 7913} 7914 7915multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 7916 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 7917 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 7918} 7919 7920//===----------------------------------------------------------------------===// 7921// SVE BFloat16 Group 7922//===----------------------------------------------------------------------===// 7923 7924class sve_bfloat_dot_base<bits<2> opc, string asm, string ops, dag iops> 7925: I<(outs ZPR32:$Zda), iops, asm, ops, "", []>, Sched<[]> { 7926 bits<5> Zda; 7927 bits<5> Zn; 7928 let Inst{31-21} = 0b01100100011; 7929 let Inst{15-14} = opc; 7930 let Inst{13-10} = 0b0000; 7931 let Inst{9-5} = Zn; 7932 let Inst{4-0} = Zda; 7933 7934 let Constraints = "$Zda = $_Zda"; 7935 let DestructiveInstType = DestructiveOther; 7936 let ElementSize = ElementSizeH; 7937} 7938 7939class sve_bfloat_dot<string asm> 7940: sve_bfloat_dot_base<0b10, asm, "\t$Zda, $Zn, $Zm", 7941 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm)> { 7942 bits<5> Zm; 7943 let Inst{20-16} = Zm; 7944} 7945 7946multiclass sve_bfloat_dot<string asm, SDPatternOperator op> { 7947 def NAME : sve_bfloat_dot<asm>; 7948 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7949} 7950 7951class sve_bfloat_dot_indexed<string asm> 7952: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 7953 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS:$iop)> { 7954 bits<2> iop; 7955 bits<3> Zm; 7956 let Inst{20-19} = iop; 7957 let Inst{18-16} = Zm; 7958} 7959 7960multiclass sve_bfloat_dot_indexed<string asm, SDPatternOperator op> { 7961 def NAME : sve_bfloat_dot_indexed<asm>; 7962 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexS_timm, !cast<Instruction>(NAME)>; 7963} 7964 7965class sve_bfloat_matmul<string asm> 7966: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 7967 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 7968 bits<5> Zm; 7969 bits<5> Zda; 7970 bits<5> Zn; 7971 let Inst{31-21} = 0b01100100011; 7972 let Inst{20-16} = Zm; 7973 let Inst{15-10} = 0b111001; 7974 let Inst{9-5} = Zn; 7975 let Inst{4-0} = Zda; 7976 7977 let Constraints = "$Zda = $_Zda"; 7978 let DestructiveInstType = DestructiveOther; 7979 let ElementSize = ElementSizeH; 7980} 7981 7982multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 7983 def NAME : sve_bfloat_matmul<asm>; 7984 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7985} 7986 7987class sve_bfloat_matmul_longvecl<bit BT, string asm> 7988: sve_bfloat_matmul<asm> { 7989 let Inst{23} = 0b1; 7990 let Inst{14-13} = 0b00; 7991 let Inst{10} = BT; 7992} 7993 7994multiclass sve_bfloat_matmul_longvecl<bit BT, string asm, SDPatternOperator op> { 7995 def NAME : sve_bfloat_matmul_longvecl<BT, asm>; 7996 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 7997} 7998 7999class sve_bfloat_matmul_longvecl_idx<bit BT, string asm> 8000: sve_bfloat_dot_base<0b01, asm, "\t$Zda, $Zn, $Zm$iop", 8001 (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexH:$iop)> { 8002 bits<3> iop; 8003 bits<3> Zm; 8004 let Inst{23} = 0b1; 8005 let Inst{20-19} = iop{2-1}; 8006 let Inst{18-16} = Zm; 8007 let Inst{11} = iop{0}; 8008 let Inst{10} = BT; 8009} 8010 8011multiclass sve_bfloat_matmul_longvecl_idx<bit BT, string asm, SDPatternOperator op> { 8012 def NAME : sve_bfloat_matmul_longvecl_idx<BT, asm>; 8013 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16, i64, VectorIndexH_timm, !cast<Instruction>(NAME)>; 8014} 8015 8016class sve_bfloat_convert<bit N, string asm> 8017: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 8018 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 8019 bits<5> Zd; 8020 bits<3> Pg; 8021 bits<5> Zn; 8022 let Inst{31-25} = 0b0110010; 8023 let Inst{24} = N; 8024 let Inst{23-13} = 0b10001010101; 8025 let Inst{12-10} = Pg; 8026 let Inst{9-5} = Zn; 8027 let Inst{4-0} = Zd; 8028 8029 let Constraints = "$Zd = $_Zd"; 8030 let DestructiveInstType = DestructiveOther; 8031 let hasSideEffects = 1; 8032 let ElementSize = ElementSizeS; 8033} 8034 8035multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 8036 def NAME : sve_bfloat_convert<N, asm>; 8037 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 8038} 8039 8040//===----------------------------------------------------------------------===// 8041// SVE Integer Matrix Multiply Group 8042//===----------------------------------------------------------------------===// 8043 8044class sve_int_matmul<bits<2> uns, string asm> 8045: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8046 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8047 bits<5> Zda; 8048 bits<5> Zn; 8049 bits<5> Zm; 8050 let Inst{31-24} = 0b01000101; 8051 let Inst{23-22} = uns; 8052 let Inst{21} = 0; 8053 let Inst{20-16} = Zm; 8054 let Inst{15-10} = 0b100110; 8055 let Inst{9-5} = Zn; 8056 let Inst{4-0} = Zda; 8057 8058 let Constraints = "$Zda = $_Zda"; 8059 let DestructiveInstType = DestructiveOther; 8060 let ElementSize = ZPR32.ElementSize; 8061} 8062 8063multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 8064 def NAME : sve_int_matmul<uns, asm>; 8065 8066 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8067} 8068 8069//===----------------------------------------------------------------------===// 8070// SVE Integer Dot Product Mixed Sign Group 8071//===----------------------------------------------------------------------===// 8072 8073class sve_int_dot_mixed<string asm> 8074: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8075 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8076 bits<5> Zda; 8077 bits<5> Zn; 8078 bits<5> Zm; 8079 let Inst{31-21} = 0b01000100100; 8080 let Inst{20-16} = Zm; 8081 let Inst{15-10} = 0b011110; 8082 let Inst{9-5} = Zn; 8083 let Inst{4-0} = Zda; 8084 8085 let Constraints = "$Zda = $_Zda"; 8086 let DestructiveInstType = DestructiveOther; 8087 let ElementSize = ZPR32.ElementSize; 8088} 8089 8090multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 8091 def NAME : sve_int_dot_mixed<asm>; 8092 8093 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8094} 8095 8096//===----------------------------------------------------------------------===// 8097// SVE Integer Dot Product Mixed Sign - Indexed Group 8098//===----------------------------------------------------------------------===// 8099 8100class sve_int_dot_mixed_indexed<bit U, string asm> 8101: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 8102 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 8103 bits<5> Zda; 8104 bits<5> Zn; 8105 bits<3> Zm; 8106 bits<2> idx; 8107 let Inst{31-21} = 0b01000100101; 8108 let Inst{20-19} = idx; 8109 let Inst{18-16} = Zm; 8110 let Inst{15-11} = 0b00011; 8111 let Inst{10} = U; 8112 let Inst{9-5} = Zn; 8113 let Inst{4-0} = Zda; 8114 8115 let Constraints = "$Zda = $_Zda"; 8116 let DestructiveInstType = DestructiveOther; 8117 let ElementSize = ZPR32.ElementSize; 8118} 8119 8120multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 8121 def NAME : sve_int_dot_mixed_indexed<U, asm>; 8122 8123 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8124} 8125 8126//===----------------------------------------------------------------------===// 8127// SVE Floating Point Matrix Multiply Accumulate Group 8128//===----------------------------------------------------------------------===// 8129 8130class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 8131: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 8132 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8133 bits<5> Zda; 8134 bits<5> Zn; 8135 bits<5> Zm; 8136 let Inst{31-23} = 0b011001001; 8137 let Inst{22} = sz; 8138 let Inst{21} = 1; 8139 let Inst{20-16} = Zm; 8140 let Inst{15-10} = 0b111001; 8141 let Inst{9-5} = Zn; 8142 let Inst{4-0} = Zda; 8143 8144 let Constraints = "$Zda = $_Zda"; 8145 let DestructiveInstType = DestructiveOther; 8146 let ElementSize = zprty.ElementSize; 8147} 8148 8149multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 8150 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 8151 8152 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 8153} 8154 8155//===----------------------------------------------------------------------===// 8156// SVE Memory - Contiguous Load And Replicate 256-bit Group 8157//===----------------------------------------------------------------------===// 8158 8159class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 8160: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 8161 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 8162 bits<5> Zt; 8163 bits<5> Rn; 8164 bits<3> Pg; 8165 bits<4> imm4; 8166 let Inst{31-25} = 0b1010010; 8167 let Inst{24-23} = sz; 8168 let Inst{22-20} = 0b010; 8169 let Inst{19-16} = imm4; 8170 let Inst{15-13} = 0b001; 8171 let Inst{12-10} = Pg; 8172 let Inst{9-5} = Rn; 8173 let Inst{4-0} = Zt; 8174 8175 let mayLoad = 1; 8176} 8177 8178multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 8179 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 8180 def NAME : sve_mem_ldor_si<sz, asm, listty>; 8181 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8182 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 8183 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8184 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 8185 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 8186 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 8187 8188 // Base addressing mode 8189 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 8190 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 8191 let AddedComplexity = 2 in { 8192 // Reg + Imm addressing mode 8193 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 8194 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 8195 } 8196} 8197 8198class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 8199 RegisterOperand gprty> 8200: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 8201 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 8202 bits<5> Zt; 8203 bits<3> Pg; 8204 bits<5> Rn; 8205 bits<5> Rm; 8206 let Inst{31-25} = 0b1010010; 8207 let Inst{24-23} = sz; 8208 let Inst{22-21} = 0b01; 8209 let Inst{20-16} = Rm; 8210 let Inst{15-13} = 0; 8211 let Inst{12-10} = Pg; 8212 let Inst{9-5} = Rn; 8213 let Inst{4-0} = Zt; 8214 8215 let mayLoad = 1; 8216} 8217 8218multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 8219 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 8220 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 8221 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 8222 8223 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 8224 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 8225 8226 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 8227 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 8228} 8229 8230//===----------------------------------------------------------------------===// 8231// SVE Interleave 128-bit Elements Group 8232//===----------------------------------------------------------------------===// 8233 8234class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 8235: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 8236 asm, "\t$Zd, $Zn, $Zm", 8237 "", 8238 []>, Sched<[]> { 8239 bits<5> Zd; 8240 bits<5> Zm; 8241 bits<5> Zn; 8242 let Inst{31-21} = 0b00000101101; 8243 let Inst{20-16} = Zm; 8244 let Inst{15-13} = 0b000; 8245 let Inst{12-11} = opc; 8246 let Inst{10} = P; 8247 let Inst{9-5} = Zn; 8248 let Inst{4-0} = Zd; 8249} 8250 8251multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 8252 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 8253 8254 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8255 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 8256 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 8257 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 8258 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 8259 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 8260 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 8261 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 8262} 8263 8264/// Addressing modes 8265def am_sve_indexed_s4 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 8266def am_sve_indexed_s6 :ComplexPattern<i64, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 8267 8268def am_sve_regreg_lsl0 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<0>", []>; 8269def am_sve_regreg_lsl1 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<1>", []>; 8270def am_sve_regreg_lsl2 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<2>", []>; 8271def am_sve_regreg_lsl3 : ComplexPattern<i64, 2, "SelectSVERegRegAddrMode<3>", []>; 8272 8273// Predicated pseudo floating point two operand instructions. 8274multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 8275 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8276 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8277 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8278 8279 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8280 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8281 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8282 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8283 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8284 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8285} 8286 8287// Predicated pseudo integer two operand instructions. 8288multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 8289 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8290 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8291 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8292 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8293 8294 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8295 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8296 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8297 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8298} 8299 8300// As sve_int_bin_pred but when only i32 and i64 vector types are required. 8301multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 8302 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8303 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8304 8305 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8306 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8307} 8308 8309// Predicated pseudo integer two operand instructions. Second operand is an 8310// immediate specified by imm_[bhsd]. 8311multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 8312 ComplexPattern imm_b, ComplexPattern imm_h, 8313 ComplexPattern imm_s, ComplexPattern imm_d> { 8314 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 8315 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 8316 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 8317 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 8318 8319 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 8320 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 8321 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 8322 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8323} 8324 8325multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> { 8326 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8327 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8328 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8329 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8330 8331 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8332 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8333 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8334 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8335} 8336