1//=-- SVEInstrFormats.td - AArch64 SVE Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Vector Extension (SVE) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def SDT_AArch64Setcc : SDTypeProfile<1, 4, [ 14 SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVec<3>, 15 SDTCVecEltisVT<0, i1>, SDTCVecEltisVT<1, i1>, SDTCisSameAs<2, 3>, 16 SDTCisVT<4, OtherVT> 17]>; 18 19def AArch64setcc_z : SDNode<"AArch64ISD::SETCC_MERGE_ZERO", SDT_AArch64Setcc>; 20 21def SVEPatternOperand : AsmOperandClass { 22 let Name = "SVEPattern"; 23 let ParserMethod = "tryParseSVEPattern"; 24 let PredicateMethod = "isSVEPattern"; 25 let RenderMethod = "addImmOperands"; 26 let DiagnosticType = "InvalidSVEPattern"; 27} 28 29def sve_pred_enum : Operand<i32>, TImmLeaf<i32, [{ 30 return (((uint32_t)Imm) < 32); 31 }]> { 32 33 let PrintMethod = "printSVEPattern"; 34 let ParserMatchClass = SVEPatternOperand; 35} 36 37def SVEVecLenSpecifierOperand : AsmOperandClass { 38 let Name = "SVEVecLenSpecifier"; 39 let ParserMethod = "tryParseSVEVecLenSpecifier"; 40 let PredicateMethod = "isSVEVecLenSpecifier"; 41 let RenderMethod = "addImmOperands"; 42 let DiagnosticType = "InvalidSVEVecLenSpecifier"; 43} 44 45def sve_vec_len_specifier_enum : Operand<i32>, TImmLeaf<i32, [{ 46 return (((uint32_t)Imm) < 2); 47 }]> { 48 49 let PrintMethod = "printSVEVecLenSpecifier"; 50 let ParserMatchClass = SVEVecLenSpecifierOperand; 51} 52 53def SVEPrefetchOperand : AsmOperandClass { 54 let Name = "SVEPrefetch"; 55 let ParserMethod = "tryParsePrefetch<true>"; 56 let PredicateMethod = "isPrefetch"; 57 let RenderMethod = "addPrefetchOperands"; 58} 59 60def sve_prfop : Operand<i32>, TImmLeaf<i32, [{ 61 return (((uint32_t)Imm) <= 15); 62 }]> { 63 let PrintMethod = "printPrefetchOp<true>"; 64 let ParserMatchClass = SVEPrefetchOperand; 65} 66 67class SVELogicalImmOperand<int Width> : AsmOperandClass { 68 let Name = "SVELogicalImm" # Width; 69 let DiagnosticType = "LogicalSecondSource"; 70 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 71 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 72} 73 74def sve_logical_imm8 : Operand<i64> { 75 let ParserMatchClass = SVELogicalImmOperand<8>; 76 let PrintMethod = "printLogicalImm<int8_t>"; 77 78 let MCOperandPredicate = [{ 79 if (!MCOp.isImm()) 80 return false; 81 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 82 return AArch64_AM::isSVEMaskOfIdenticalElements<int8_t>(Val); 83 }]; 84} 85 86def sve_logical_imm16 : Operand<i64> { 87 let ParserMatchClass = SVELogicalImmOperand<16>; 88 let PrintMethod = "printLogicalImm<int16_t>"; 89 90 let MCOperandPredicate = [{ 91 if (!MCOp.isImm()) 92 return false; 93 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 94 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val); 95 }]; 96} 97 98def sve_logical_imm32 : Operand<i64> { 99 let ParserMatchClass = SVELogicalImmOperand<32>; 100 let PrintMethod = "printLogicalImm<int32_t>"; 101 102 let MCOperandPredicate = [{ 103 if (!MCOp.isImm()) 104 return false; 105 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 106 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val); 107 }]; 108} 109 110class SVEPreferredLogicalImmOperand<int Width> : AsmOperandClass { 111 let Name = "SVEPreferredLogicalImm" # Width; 112 let PredicateMethod = "isSVEPreferredLogicalImm<int" # Width # "_t>"; 113 let RenderMethod = "addLogicalImmOperands<int" # Width # "_t>"; 114} 115 116def sve_preferred_logical_imm16 : Operand<i64> { 117 let ParserMatchClass = SVEPreferredLogicalImmOperand<16>; 118 let PrintMethod = "printSVELogicalImm<int16_t>"; 119 120 let MCOperandPredicate = [{ 121 if (!MCOp.isImm()) 122 return false; 123 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 124 return AArch64_AM::isSVEMaskOfIdenticalElements<int16_t>(Val) && 125 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 126 }]; 127} 128 129def sve_preferred_logical_imm32 : Operand<i64> { 130 let ParserMatchClass = SVEPreferredLogicalImmOperand<32>; 131 let PrintMethod = "printSVELogicalImm<int32_t>"; 132 133 let MCOperandPredicate = [{ 134 if (!MCOp.isImm()) 135 return false; 136 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 137 return AArch64_AM::isSVEMaskOfIdenticalElements<int32_t>(Val) && 138 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 139 }]; 140} 141 142def sve_preferred_logical_imm64 : Operand<i64> { 143 let ParserMatchClass = SVEPreferredLogicalImmOperand<64>; 144 let PrintMethod = "printSVELogicalImm<int64_t>"; 145 146 let MCOperandPredicate = [{ 147 if (!MCOp.isImm()) 148 return false; 149 int64_t Val = AArch64_AM::decodeLogicalImmediate(MCOp.getImm(), 64); 150 return AArch64_AM::isSVEMaskOfIdenticalElements<int64_t>(Val) && 151 AArch64_AM::isSVEMoveMaskPreferredLogicalImmediate(Val); 152 }]; 153} 154 155class SVELogicalImmNotOperand<int Width> : AsmOperandClass { 156 let Name = "SVELogicalImm" # Width # "Not"; 157 let DiagnosticType = "LogicalSecondSource"; 158 let PredicateMethod = "isLogicalImm<int" # Width # "_t>"; 159 let RenderMethod = "addLogicalImmNotOperands<int" # Width # "_t>"; 160} 161 162def sve_logical_imm8_not : Operand<i64> { 163 let ParserMatchClass = SVELogicalImmNotOperand<8>; 164} 165 166def sve_logical_imm16_not : Operand<i64> { 167 let ParserMatchClass = SVELogicalImmNotOperand<16>; 168} 169 170def sve_logical_imm32_not : Operand<i64> { 171 let ParserMatchClass = SVELogicalImmNotOperand<32>; 172} 173 174class SVEShiftedImmOperand<int ElementWidth, string Infix, string Predicate> 175 : AsmOperandClass { 176 let Name = "SVE" # Infix # "Imm" # ElementWidth; 177 let DiagnosticType = "Invalid" # Name; 178 let RenderMethod = "addImmWithOptionalShiftOperands<8>"; 179 let ParserMethod = "tryParseImmWithOptionalShift"; 180 let PredicateMethod = Predicate; 181} 182 183def SVECpyImmOperand8 : SVEShiftedImmOperand<8, "Cpy", "isSVECpyImm<int8_t>">; 184def SVECpyImmOperand16 : SVEShiftedImmOperand<16, "Cpy", "isSVECpyImm<int16_t>">; 185def SVECpyImmOperand32 : SVEShiftedImmOperand<32, "Cpy", "isSVECpyImm<int32_t>">; 186def SVECpyImmOperand64 : SVEShiftedImmOperand<64, "Cpy", "isSVECpyImm<int64_t>">; 187 188def SVEAddSubImmOperand8 : SVEShiftedImmOperand<8, "AddSub", "isSVEAddSubImm<int8_t>">; 189def SVEAddSubImmOperand16 : SVEShiftedImmOperand<16, "AddSub", "isSVEAddSubImm<int16_t>">; 190def SVEAddSubImmOperand32 : SVEShiftedImmOperand<32, "AddSub", "isSVEAddSubImm<int32_t>">; 191def SVEAddSubImmOperand64 : SVEShiftedImmOperand<64, "AddSub", "isSVEAddSubImm<int64_t>">; 192 193class imm8_opt_lsl<int ElementWidth, string printType, 194 AsmOperandClass OpndClass> 195 : Operand<i32> { 196 let EncoderMethod = "getImm8OptLsl"; 197 let DecoderMethod = "DecodeImm8OptLsl<" # ElementWidth # ">"; 198 let PrintMethod = "printImm8OptLsl<" # printType # ">"; 199 let ParserMatchClass = OpndClass; 200 let MIOperandInfo = (ops i32imm, i32imm); 201} 202 203def cpy_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "int8_t", SVECpyImmOperand8>; 204def cpy_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "int16_t", SVECpyImmOperand16>; 205def cpy_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "int32_t", SVECpyImmOperand32>; 206def cpy_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "int64_t", SVECpyImmOperand64>; 207 208def addsub_imm8_opt_lsl_i8 : imm8_opt_lsl<8, "uint8_t", SVEAddSubImmOperand8>; 209def addsub_imm8_opt_lsl_i16 : imm8_opt_lsl<16, "uint16_t", SVEAddSubImmOperand16>; 210def addsub_imm8_opt_lsl_i32 : imm8_opt_lsl<32, "uint32_t", SVEAddSubImmOperand32>; 211def addsub_imm8_opt_lsl_i64 : imm8_opt_lsl<64, "uint64_t", SVEAddSubImmOperand64>; 212 213def SVEAddSubImm8Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i8>", []>; 214def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", []>; 215def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>; 216def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>; 217 218def SVECpyDupImm8Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>; 219def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>; 220def SVECpyDupImm32Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i32>", []>; 221def SVECpyDupImm64Pat : ComplexPattern<i64, 2, "SelectSVECpyDupImm<MVT::i64>", []>; 222 223def SVELogicalImm8Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8>", []>; 224def SVELogicalImm16Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16>", []>; 225def SVELogicalImm32Pat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32>", []>; 226def SVELogicalImm64Pat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64>", []>; 227 228def SVELogicalImm8NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i8, true>", []>; 229def SVELogicalImm16NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i16, true>", []>; 230def SVELogicalImm32NotPat : ComplexPattern<i32, 1, "SelectSVELogicalImm<MVT::i32, true>", []>; 231def SVELogicalImm64NotPat : ComplexPattern<i64, 1, "SelectSVELogicalImm<MVT::i64, true>", []>; 232 233def SVEArithUImm8Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i8>", []>; 234def SVEArithUImm16Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i16>", []>; 235def SVEArithUImm32Pat : ComplexPattern<i32, 1, "SelectSVEArithImm<MVT::i32>", []>; 236def SVEArithUImm64Pat : ComplexPattern<i64, 1, "SelectSVEArithImm<MVT::i64>", []>; 237 238def SVEArithSImmPat32 : ComplexPattern<i32, 1, "SelectSVESignedArithImm", []>; 239def SVEArithSImmPat64 : ComplexPattern<i64, 1, "SelectSVESignedArithImm", []>; 240 241def SVEShiftImmL8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 7>", []>; 242def SVEShiftImmL16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 15>", []>; 243def SVEShiftImmL32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<0, 31>", []>; 244def SVEShiftImmL64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<0, 63>", []>; 245def SVEShiftImmR8 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 8, true>", []>; 246def SVEShiftImmR16 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 16, true>", []>; 247def SVEShiftImmR32 : ComplexPattern<i32, 1, "SelectSVEShiftImm<1, 32, true>", []>; 248def SVEShiftImmR64 : ComplexPattern<i64, 1, "SelectSVEShiftImm<1, 64, true>", []>; 249 250def SVEShiftSplatImmR : ComplexPattern<iAny, 1, "SelectSVEShiftSplatImmR", []>; 251 252def SVEAllActive : ComplexPattern<untyped, 0, "SelectAllActivePredicate", []>; 253 254class SVEExactFPImm<string Suffix, string ValA, string ValB> : AsmOperandClass { 255 let Name = "SVEExactFPImmOperand" # Suffix; 256 let DiagnosticType = "Invalid" # Name; 257 let ParserMethod = "tryParseFPImm<false>"; 258 let PredicateMethod = "isExactFPImm<" # ValA # ", " # ValB # ">"; 259 let RenderMethod = "addExactFPImmOperands<" # ValA # ", " # ValB # ">"; 260} 261 262class SVEExactFPImmOperand<string Suffix, string ValA, string ValB> : Operand<i32> { 263 let PrintMethod = "printExactFPImm<" # ValA # ", " # ValB # ">"; 264 let ParserMatchClass = SVEExactFPImm<Suffix, ValA, ValB>; 265} 266 267def sve_fpimm_half_one 268 : SVEExactFPImmOperand<"HalfOne", "AArch64ExactFPImm::half", 269 "AArch64ExactFPImm::one">; 270def sve_fpimm_half_two 271 : SVEExactFPImmOperand<"HalfTwo", "AArch64ExactFPImm::half", 272 "AArch64ExactFPImm::two">; 273def sve_fpimm_zero_one 274 : SVEExactFPImmOperand<"ZeroOne", "AArch64ExactFPImm::zero", 275 "AArch64ExactFPImm::one">; 276 277def sve_incdec_imm : Operand<i32>, TImmLeaf<i32, [{ 278 return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17); 279}]> { 280 let ParserMatchClass = Imm1_16Operand; 281 let EncoderMethod = "getSVEIncDecImm"; 282 let DecoderMethod = "DecodeSVEIncDecImm"; 283} 284 285// This allows i32 immediate extraction from i64 based arithmetic. 286def sve_cnt_mul_imm_i32 : ComplexPattern<i32, 1, "SelectCntImm<1, 16, 1, false>">; 287def sve_cnt_mul_imm_i64 : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, false>">; 288def sve_cnt_shl_imm : ComplexPattern<i64, 1, "SelectCntImm<1, 16, 1, true>">; 289 290def sve_ext_imm_0_31 : ComplexPattern<i64, 1, "SelectEXTImm<31, 8>">; 291def sve_ext_imm_0_63 : ComplexPattern<i64, 1, "SelectEXTImm<63, 4>">; 292def sve_ext_imm_0_127 : ComplexPattern<i64, 1, "SelectEXTImm<127, 2>">; 293def sve_ext_imm_0_255 : ComplexPattern<i64, 1, "SelectEXTImm<255, 1>">; 294 295def int_aarch64_sve_cntp_oneuse : PatFrag<(ops node:$pred, node:$src2), 296 (int_aarch64_sve_cntp node:$pred, node:$src2), [{ 297 return N->hasOneUse(); 298}]>; 299 300def step_vector_oneuse : PatFrag<(ops node:$idx), 301 (step_vector node:$idx), [{ 302 return N->hasOneUse(); 303}]>; 304 305 306//===----------------------------------------------------------------------===// 307// SVE PTrue - These are used extensively throughout the pattern matching so 308// it's important we define them first. 309//===----------------------------------------------------------------------===// 310 311class sve_int_ptrue<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 312 ValueType vt, SDPatternOperator op> 313: I<(outs pprty:$Pd), (ins sve_pred_enum:$pattern), 314 asm, "\t$Pd, $pattern", 315 "", 316 [(set (vt pprty:$Pd), (op sve_pred_enum:$pattern))]>, Sched<[]> { 317 bits<4> Pd; 318 bits<5> pattern; 319 let Inst{31-24} = 0b00100101; 320 let Inst{23-22} = sz8_64; 321 let Inst{21-19} = 0b011; 322 let Inst{18-17} = opc{2-1}; 323 let Inst{16} = opc{0}; 324 let Inst{15-10} = 0b111000; 325 let Inst{9-5} = pattern; 326 let Inst{4} = 0b0; 327 let Inst{3-0} = Pd; 328 329 let Defs = !if(!eq (opc{0}, 1), [NZCV], []); 330 let ElementSize = pprty.ElementSize; 331 let isReMaterializable = 1; 332} 333 334multiclass sve_int_ptrue<bits<3> opc, string asm, SDPatternOperator op> { 335 def _B : sve_int_ptrue<0b00, opc, asm, PPR8, nxv16i1, op>; 336 def _H : sve_int_ptrue<0b01, opc, asm, PPR16, nxv8i1, op>; 337 def _S : sve_int_ptrue<0b10, opc, asm, PPR32, nxv4i1, op>; 338 def _D : sve_int_ptrue<0b11, opc, asm, PPR64, nxv2i1, op>; 339 340 def : InstAlias<asm # "\t$Pd", 341 (!cast<Instruction>(NAME # _B) PPR8:$Pd, 0b11111), 1>; 342 def : InstAlias<asm # "\t$Pd", 343 (!cast<Instruction>(NAME # _H) PPR16:$Pd, 0b11111), 1>; 344 def : InstAlias<asm # "\t$Pd", 345 (!cast<Instruction>(NAME # _S) PPR32:$Pd, 0b11111), 1>; 346 def : InstAlias<asm # "\t$Pd", 347 (!cast<Instruction>(NAME # _D) PPR64:$Pd, 0b11111), 1>; 348} 349 350def SDT_AArch64PTrue : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVT<1, i32>]>; 351def AArch64ptrue : SDNode<"AArch64ISD::PTRUE", SDT_AArch64PTrue>; 352 353let Predicates = [HasSVEorSME] in { 354 defm PTRUE : sve_int_ptrue<0b000, "ptrue", AArch64ptrue>; 355 defm PTRUES : sve_int_ptrue<0b001, "ptrues", null_frag>; 356 357 def : Pat<(nxv16i1 immAllOnesV), (PTRUE_B 31)>; 358 def : Pat<(nxv8i1 immAllOnesV), (PTRUE_H 31)>; 359 def : Pat<(nxv4i1 immAllOnesV), (PTRUE_S 31)>; 360 def : Pat<(nxv2i1 immAllOnesV), (PTRUE_D 31)>; 361} 362 363//===----------------------------------------------------------------------===// 364// SVE pattern match helpers. 365//===----------------------------------------------------------------------===// 366 367class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 368 Instruction inst> 369: Pat<(vtd (op vt1:$Op1)), 370 (inst $Op1)>; 371 372class SVE_1_Op_Passthru_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 373 ValueType vts, Instruction inst> 374: Pat<(vtd (op pg:$Op1, vts:$Op2, vtd:$Op3)), 375 (inst $Op3, $Op1, $Op2)>; 376 377 378multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 379 ValueType vts, Instruction inst> { 380 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd undef))), 381 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 382 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, vtd:$Op3)), 383 (inst $Op3, $Op1, $Op2)>; 384} 385 386// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the 387// type of rounding. This is matched by timm0_1 in pattern below and ignored. 388class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 389 ValueType vts, Instruction inst> 390: Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), vtd:$Op3)), 391 (inst $Op3, $Op1, $Op2)>; 392 393multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg, 394 ValueType vts, Instruction inst>{ 395 def : Pat<(vtd (op pg:$Op1, vts:$Op2, (i64 timm0_1), (vtd undef))), 396 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 397 def : Pat<(vtd (op (pg (SVEAllActive:$Op1)), vts:$Op2, (i64 timm0_1), vtd:$Op3)), 398 (inst $Op3, $Op1, $Op2)>; 399} 400 401class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 402 ValueType it, ComplexPattern cpx, Instruction inst> 403 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm, i32:$shift)))))), 404 (inst $Op1, i32:$imm, i32:$shift)>; 405 406class SVE_1_Op_Imm_Arith_All_Active<ValueType vt, ValueType pt, SDPatternOperator op, 407 ZPRRegOp zprty, ValueType it, ComplexPattern cpx, Instruction inst> 408 : Pat<(vt (op (pt (SVEAllActive)), (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm)))))), 409 (inst $Op1, i32:$imm)>; 410 411class SVE_1_Op_Imm_Log_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty, 412 ValueType it, ComplexPattern cpx, Instruction inst> 413 : Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i64:$imm)))))), 414 (inst $Op1, i64:$imm)>; 415 416class SVE_2_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 417 ValueType vt2, Instruction inst> 418: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 419 (inst $Op1, $Op2)>; 420 421class SVE_2_Op_Pred_All_Active<ValueType vtd, SDPatternOperator op, 422 ValueType pt, ValueType vt1, ValueType vt2, 423 Instruction inst> 424: Pat<(vtd (op (pt (SVEAllActive)), vt1:$Op1, vt2:$Op2)), 425 (inst $Op1, $Op2)>; 426 427class SVE_2_Op_Pred_All_Active_Pt<ValueType vtd, SDPatternOperator op, 428 ValueType pt, ValueType vt1, ValueType vt2, 429 Instruction inst> 430: Pat<(vtd (op (pt (SVEAllActive:$Op1)), vt1:$Op2, vt2:$Op3)), 431 (inst $Op1, $Op2, $Op3)>; 432 433class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 434 ValueType vt2, ValueType vt3, Instruction inst> 435: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 436 (inst $Op1, $Op2, $Op3)>; 437 438multiclass SVE_3_Op_Undef_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 439 ValueType vt2, ValueType vt3, Instruction inst> { 440 def : Pat<(vtd (op (vt1 undef), vt2:$Op1, vt3:$Op2)), 441 (inst (IMPLICIT_DEF), $Op1, $Op2)>; 442 def : Pat<(vtd (op vt1:$Op1, (vt2 (SVEAllActive:$Op2)), vt3:$Op3)), 443 (inst $Op1, $Op2, $Op3)>; 444} 445 446class SVE_4_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 447 ValueType vt2, ValueType vt3, ValueType vt4, 448 Instruction inst> 449: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, vt4:$Op4)), 450 (inst $Op1, $Op2, $Op3, $Op4)>; 451 452class SVE_2_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 453 ValueType vt2, Operand ImmTy, Instruction inst> 454: Pat<(vtd (op vt1:$Op1, (vt2 ImmTy:$Op2))), 455 (inst $Op1, ImmTy:$Op2)>; 456 457class SVE_3_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 458 ValueType vt2, ValueType vt3, Operand ImmTy, 459 Instruction inst> 460: Pat<(vtd (op vt1:$Op1, vt2:$Op2, (vt3 ImmTy:$Op3))), 461 (inst $Op1, $Op2, ImmTy:$Op3)>; 462 463class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 464 ValueType vt2, ValueType vt3, ValueType vt4, 465 Operand ImmTy, Instruction inst> 466: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))), 467 (inst $Op1, $Op2, $Op3, ImmTy:$Op4)>; 468 469def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>; 470def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>; 471 472let AddedComplexity = 1 in { 473class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1, 474 ValueType vt2, ValueType vt3, Instruction inst> 475: Pat<(vtd (vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), vt3:$Op3))), 476 (inst $Op1, $Op2, $Op3)>; 477 478class SVE_3_Op_Pat_Shift_Imm_SelZero<ValueType vtd, SDPatternOperator op, 479 ValueType vt1, ValueType vt2, 480 Operand vt3, Instruction inst> 481: Pat<(vtd (op vt1:$Op1, (vselect vt1:$Op1, vt2:$Op2, (SVEDup0)), (i32 (vt3:$Op3)))), 482 (inst $Op1, $Op2, vt3:$Op3)>; 483} 484 485// 486// Common but less generic patterns. 487// 488 489class SVE_1_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 490 Instruction inst, Instruction ptrue> 491: Pat<(vtd (op vt1:$Op1)), 492 (inst (IMPLICIT_DEF), (ptrue 31), $Op1)>; 493 494class SVE_2_Op_AllActive_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 495 ValueType vt2, Instruction inst, Instruction ptrue> 496: Pat<(vtd (op vt1:$Op1, vt2:$Op2)), 497 (inst (ptrue 31), $Op1, $Op2)>; 498 499class SVE_InReg_Extend<ValueType vt, SDPatternOperator op, ValueType pt, 500 ValueType inreg_vt, Instruction inst> 501: Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, vt:$PassThru)), 502 (inst $PassThru, $Pg, $Src)>; 503 504multiclass SVE_InReg_Extend_PassthruUndef<ValueType vt, SDPatternOperator op, ValueType pt, 505 ValueType inreg_vt, Instruction inst> { 506 def : Pat<(vt (op pt:$Pg, vt:$Src, inreg_vt, (vt undef))), 507 (inst (IMPLICIT_DEF), $Pg, $Src)>; 508 def : Pat<(vt (op (pt (SVEAllActive:$Pg)), vt:$Src, inreg_vt, vt:$PassThru)), 509 (inst $PassThru, $Pg, $Src)>; 510} 511 512class SVE_Shift_DupImm_Pred_Pat<ValueType vt, SDPatternOperator op, 513 ValueType pt, ValueType it, 514 ComplexPattern cast, Instruction inst> 515: Pat<(vt (op pt:$Pg, vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 516 (inst $Pg, $Rn, i32:$imm)>; 517 518class SVE_Shift_DupImm_All_Active_Pat<ValueType vt, SDPatternOperator op, 519 ValueType pt, ValueType it, 520 ComplexPattern cast, Instruction inst> 521: Pat<(vt (op (pt (SVEAllActive)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), 522 (inst $Rn, i32:$imm)>; 523 524class SVE_2_Op_Fp_Imm_Pat<ValueType vt, SDPatternOperator op, 525 ValueType pt, ValueType it, 526 FPImmLeaf immL, int imm, 527 Instruction inst> 528: Pat<(vt (op (pt PPR_3b:$Pg), (vt ZPR:$Zs1), (vt (splat_vector (it immL))))), 529 (inst $Pg, $Zs1, imm)>; 530 531class SVE_2_Op_Fp_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, 532 ValueType pt, ValueType it, 533 FPImmLeaf immL, int imm, 534 Instruction inst> 535: Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Zs1, (SVEDup0)), 536 (vt (splat_vector (it immL))))), 537 (inst $Pg, $Zs1, imm)>; 538 539// Used to re-order the operands of BSP when lowering to BSL. BSP has the order: 540// mask, in1, in2 whereas BSL for SVE2 has them ordered in1, in2, mask 541class SVE_3_Op_BSP_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1, 542 ValueType vt2, ValueType vt3, Instruction inst> 543: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)), 544 (inst $Op2, $Op3, $Op1)>; 545 546class SVE_Shift_Add_All_Active_Pat<ValueType vtd, SDPatternOperator op, ValueType pt, 547 ValueType vt1, ValueType vt2, ValueType vt3, 548 Instruction inst> 549: Pat<(vtd (add vt1:$Op1, (op (pt (SVEAllActive)), vt2:$Op2, vt3:$Op3))), 550 (inst $Op1, $Op2, $Op3)>; 551 552class SVE2p1_Cvt_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt> 553 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2)), 554 (!cast<Instruction>(name) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1))>; 555 556//===----------------------------------------------------------------------===// 557// SVE pattern match helpers. 558//===----------------------------------------------------------------------===// 559 560// Matches either an intrinsic, or a predicated operation with an all active predicate 561class EitherVSelectOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode> 562: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [ 563 (intrinsic node:$Pg, node:$Op1, node:$Op2), 564 (vselect node:$Pg, (sdnode (SVEAllActive), node:$Op1, node:$Op2), node:$Op1), 565 ]>; 566 567// 568// Pseudo -> Instruction mappings 569// 570def getSVEPseudoMap : InstrMapping { 571 let FilterClass = "SVEPseudo2Instr"; 572 let RowFields = ["PseudoName"]; 573 let ColFields = ["IsInstr"]; 574 let KeyCol = ["0"]; 575 let ValueCols = [["1"]]; 576} 577 578class SVEPseudo2Instr<string name, bit instr> { 579 string PseudoName = name; 580 bit IsInstr = instr; 581} 582 583// Lookup e.g. DIV -> DIVR 584def getSVERevInstr : InstrMapping { 585 let FilterClass = "SVEInstr2Rev"; 586 let RowFields = ["InstrName"]; 587 let ColFields = ["isReverseInstr"]; 588 let KeyCol = ["0"]; 589 let ValueCols = [["1"]]; 590} 591 592// Lookup e.g. DIVR -> DIV 593def getSVENonRevInstr : InstrMapping { 594 let FilterClass = "SVEInstr2Rev"; 595 let RowFields = ["InstrName"]; 596 let ColFields = ["isReverseInstr"]; 597 let KeyCol = ["1"]; 598 let ValueCols = [["0"]]; 599} 600 601class SVEInstr2Rev<string name1, string name2, bit name1IsReverseInstr> { 602 string InstrName = !if(name1IsReverseInstr, name1, name2); 603 bit isReverseInstr = name1IsReverseInstr; 604} 605 606// 607// Pseudos for destructive operands 608// 609let hasNoSchedulingInfo = 1 in { 610 class PredTwoOpPseudo<string name, ZPRRegOp zprty, 611 FalseLanesEnum flags = FalseLanesNone> 612 : SVEPseudo2Instr<name, 0>, 613 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2), []> { 614 let FalseLanes = flags; 615 } 616 617 class PredTwoOpImmPseudo<string name, ZPRRegOp zprty, Operand immty, 618 FalseLanesEnum flags = FalseLanesNone> 619 : SVEPseudo2Instr<name, 0>, 620 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, immty:$imm), []> { 621 let FalseLanes = flags; 622 } 623 624 class PredThreeOpPseudo<string name, ZPRRegOp zprty, 625 FalseLanesEnum flags = FalseLanesNone> 626 : SVEPseudo2Instr<name, 0>, 627 Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> { 628 let FalseLanes = flags; 629 } 630} 631 632// 633// Pseudos for passthru operands 634// 635let hasNoSchedulingInfo = 1 in { 636 class PredOneOpPassthruPseudo<string name, ZPRRegOp zprty> 637 : SVEPseudo2Instr<name, 0>, 638 Pseudo<(outs zprty:$Zd), (ins zprty:$Passthru, PPR3bAny:$Pg, zprty:$Zs), []>; 639} 640 641//===----------------------------------------------------------------------===// 642// SVE Predicate Misc Group 643//===----------------------------------------------------------------------===// 644 645class sve_int_pfalse<bits<6> opc, string asm> 646: I<(outs PPR8:$Pd), (ins), 647 asm, "\t$Pd", 648 "", 649 []>, Sched<[]> { 650 bits<4> Pd; 651 let Inst{31-24} = 0b00100101; 652 let Inst{23-22} = opc{5-4}; 653 let Inst{21-19} = 0b011; 654 let Inst{18-16} = opc{3-1}; 655 let Inst{15-10} = 0b111001; 656 let Inst{9} = opc{0}; 657 let Inst{8-4} = 0b00000; 658 let Inst{3-0} = Pd; 659 660 let isReMaterializable = 1; 661} 662 663multiclass sve_int_pfalse<bits<6> opc, string asm> { 664 def NAME : sve_int_pfalse<opc, asm>; 665 666 def : InstAlias<"pfalse\t$Pd", (!cast<Instruction>(NAME) PNR8:$Pd), 0>; 667 668 def : Pat<(nxv16i1 immAllZerosV), (!cast<Instruction>(NAME))>; 669 def : Pat<(nxv8i1 immAllZerosV), (!cast<Instruction>(NAME))>; 670 def : Pat<(nxv4i1 immAllZerosV), (!cast<Instruction>(NAME))>; 671 def : Pat<(nxv2i1 immAllZerosV), (!cast<Instruction>(NAME))>; 672 def : Pat<(nxv1i1 immAllZerosV), (!cast<Instruction>(NAME))>; 673} 674 675class sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op> 676: I<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 677 asm, "\t$Pg, $Pn", 678 "", 679 [(op (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 680 bits<4> Pg; 681 bits<4> Pn; 682 let Inst{31-24} = 0b00100101; 683 let Inst{23-22} = opc{5-4}; 684 let Inst{21-19} = 0b010; 685 let Inst{18-16} = opc{3-1}; 686 let Inst{15-14} = 0b11; 687 let Inst{13-10} = Pg; 688 let Inst{9} = opc{0}; 689 let Inst{8-5} = Pn; 690 let Inst{4-0} = 0b00000; 691 692 let Defs = [NZCV]; 693 let isCompare = 1; 694} 695 696multiclass sve_int_ptest<bits<6> opc, string asm, SDPatternOperator op, 697 SDPatternOperator op_any> { 698 def NAME : sve_int_ptest<opc, asm, op>; 699 700 let hasNoSchedulingInfo = 1, isCompare = 1, Defs = [NZCV] in { 701 def _ANY : Pseudo<(outs), (ins PPRAny:$Pg, PPR8:$Pn), 702 [(op_any (nxv16i1 PPRAny:$Pg), (nxv16i1 PPR8:$Pn))]>, 703 PseudoInstExpansion<(!cast<Instruction>(NAME) PPRAny:$Pg, PPR8:$Pn)>; 704 } 705} 706 707class sve_int_pfirst_next<bits<2> sz8_64, bits<5> opc, string asm, 708 PPRRegOp pprty> 709: I<(outs pprty:$Pdn), (ins PPRAny:$Pg, pprty:$_Pdn), 710 asm, "\t$Pdn, $Pg, $_Pdn", 711 "", 712 []>, Sched<[]> { 713 bits<4> Pdn; 714 bits<4> Pg; 715 let Inst{31-24} = 0b00100101; 716 let Inst{23-22} = sz8_64; 717 let Inst{21-19} = 0b011; 718 let Inst{18-16} = opc{4-2}; 719 let Inst{15-11} = 0b11000; 720 let Inst{10-9} = opc{1-0}; 721 let Inst{8-5} = Pg; 722 let Inst{4} = 0; 723 let Inst{3-0} = Pdn; 724 725 let Constraints = "$Pdn = $_Pdn"; 726 let Defs = [NZCV]; 727 let isPTestLike = 1; 728 let ElementSize = pprty.ElementSize; 729} 730 731multiclass sve_int_pfirst<bits<5> opc, string asm, SDPatternOperator op> { 732 def _B : sve_int_pfirst_next<0b01, opc, asm, PPR8>; 733 734 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 735} 736 737multiclass sve_int_pnext<bits<5> opc, string asm, SDPatternOperator op> { 738 def _B : sve_int_pfirst_next<0b00, opc, asm, PPR8>; 739 def _H : sve_int_pfirst_next<0b01, opc, asm, PPR16>; 740 def _S : sve_int_pfirst_next<0b10, opc, asm, PPR32>; 741 def _D : sve_int_pfirst_next<0b11, opc, asm, PPR64>; 742 743 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 744 def : SVE_2_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 745 def : SVE_2_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 746 def : SVE_2_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 747} 748 749//===----------------------------------------------------------------------===// 750// SVE Predicate Count Group 751//===----------------------------------------------------------------------===// 752 753class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm, 754 RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty> 755: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn), 756 asm, "\t$Rdn, $Pg", 757 "", 758 []>, Sched<[]> { 759 bits<5> Rdn; 760 bits<4> Pg; 761 let Inst{31-24} = 0b00100101; 762 let Inst{23-22} = sz8_64; 763 let Inst{21-19} = 0b101; 764 let Inst{18-16} = opc{4-2}; 765 let Inst{15-11} = 0b10001; 766 let Inst{10-9} = opc{1-0}; 767 let Inst{8-5} = Pg; 768 let Inst{4-0} = Rdn; 769 770 // Signed 32bit forms require their GPR operand printed. 771 let AsmString = !if(!eq(opc{4,2-0}, 0b0000), 772 !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"), 773 !strconcat(asm, "\t$Rdn, $Pg")); 774 let Constraints = "$Rdn = $_Rdn"; 775} 776 777multiclass sve_int_count_r_s32<bits<5> opc, string asm, 778 SDPatternOperator op> { 779 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>; 780 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>; 781 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>; 782 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>; 783 784 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 785 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 786 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))))), 787 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 788 789 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 790 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 791 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))))), 792 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 793 794 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 795 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 796 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))))), 797 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 798 799 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 800 (EXTRACT_SUBREG (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32)), sub_32)>; 801 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))))), 802 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32))>; 803} 804 805multiclass sve_int_count_r_u32<bits<5> opc, string asm, 806 SDPatternOperator op> { 807 def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>; 808 def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>; 809 def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>; 810 def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>; 811 812 def : Pat<(i32 (op GPR32:$Rn, (nxv16i1 PPRAny:$Pg))), 813 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 814 def : Pat<(i32 (op GPR32:$Rn, (nxv8i1 PPRAny:$Pg))), 815 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 816 def : Pat<(i32 (op GPR32:$Rn, (nxv4i1 PPRAny:$Pg))), 817 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 818 def : Pat<(i32 (op GPR32:$Rn, (nxv2i1 PPRAny:$Pg))), 819 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 820} 821 822multiclass sve_int_count_r_x64<bits<5> opc, string asm, 823 SDPatternOperator op, 824 SDPatternOperator combine_op = null_frag> { 825 def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>; 826 def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>; 827 def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>; 828 def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>; 829 830 def : Pat<(i64 (op GPR64:$Rn, (nxv16i1 PPRAny:$Pg))), 831 (!cast<Instruction>(NAME # _B) PPRAny:$Pg, $Rn)>; 832 def : Pat<(i64 (op GPR64:$Rn, (nxv8i1 PPRAny:$Pg))), 833 (!cast<Instruction>(NAME # _H) PPRAny:$Pg, $Rn)>; 834 def : Pat<(i64 (op GPR64:$Rn, (nxv4i1 PPRAny:$Pg))), 835 (!cast<Instruction>(NAME # _S) PPRAny:$Pg, $Rn)>; 836 def : Pat<(i64 (op GPR64:$Rn, (nxv2i1 PPRAny:$Pg))), 837 (!cast<Instruction>(NAME # _D) PPRAny:$Pg, $Rn)>; 838 839 // combine_op(x, cntp(all_active, p)) ==> inst p, x 840 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 (SVEAllActive)), (nxv16i1 PPRAny:$pred)))), 841 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 842 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 (SVEAllActive)), (nxv8i1 PPRAny:$pred)))), 843 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 844 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 (SVEAllActive)), (nxv4i1 PPRAny:$pred)))), 845 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 846 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 (SVEAllActive)), (nxv2i1 PPRAny:$pred)))), 847 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 848 849 // combine_op(x, cntp(p, p)) ==> inst p, x 850 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv16i1 PPRAny:$pred), (nxv16i1 PPRAny:$pred)))), 851 (!cast<Instruction>(NAME # _B) PPRAny:$pred, $Rn)>; 852 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv8i1 PPRAny:$pred), (nxv8i1 PPRAny:$pred)))), 853 (!cast<Instruction>(NAME # _H) PPRAny:$pred, $Rn)>; 854 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv4i1 PPRAny:$pred), (nxv4i1 PPRAny:$pred)))), 855 (!cast<Instruction>(NAME # _S) PPRAny:$pred, $Rn)>; 856 def : Pat<(i64 (combine_op GPR64:$Rn, (int_aarch64_sve_cntp_oneuse (nxv2i1 PPRAny:$pred), (nxv2i1 PPRAny:$pred)))), 857 (!cast<Instruction>(NAME # _D) PPRAny:$pred, $Rn)>; 858} 859 860class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm, 861 ZPRRegOp zprty, PPRRegOp pprty> 862: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, pprty:$Pm), 863 asm, "\t$Zdn, $Pm", 864 "", 865 []>, Sched<[]> { 866 bits<4> Pm; 867 bits<5> Zdn; 868 let Inst{31-24} = 0b00100101; 869 let Inst{23-22} = sz8_64; 870 let Inst{21-19} = 0b101; 871 let Inst{18-16} = opc{4-2}; 872 let Inst{15-11} = 0b10000; 873 let Inst{10-9} = opc{1-0}; 874 let Inst{8-5} = Pm; 875 let Inst{4-0} = Zdn; 876 877 let Constraints = "$Zdn = $_Zdn"; 878 let DestructiveInstType = DestructiveOther; 879 let ElementSize = ElementSizeNone; 880} 881 882multiclass sve_int_count_v<bits<5> opc, string asm, 883 SDPatternOperator op = null_frag> { 884 def _H : sve_int_count_v<0b01, opc, asm, ZPR16, PPR16>; 885 def _S : sve_int_count_v<0b10, opc, asm, ZPR32, PPR32>; 886 def _D : sve_int_count_v<0b11, opc, asm, ZPR64, PPR64>; 887 888 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, !cast<Instruction>(NAME # _H)>; 889 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, !cast<Instruction>(NAME # _S)>; 890 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, !cast<Instruction>(NAME # _D)>; 891 892 def : InstAlias<asm # "\t$Zdn, $Pm", 893 (!cast<Instruction>(NAME # "_H") ZPR16:$Zdn, PPRAny:$Pm), 0>; 894 def : InstAlias<asm # "\t$Zdn, $Pm", 895 (!cast<Instruction>(NAME # "_S") ZPR32:$Zdn, PPRAny:$Pm), 0>; 896 def : InstAlias<asm # "\t$Zdn, $Pm", 897 (!cast<Instruction>(NAME # "_D") ZPR64:$Zdn, PPRAny:$Pm), 0>; 898} 899 900class sve_int_pcount_pred<bits<2> sz8_64, bits<4> opc, string asm, 901 PPRRegOp pprty> 902: I<(outs GPR64:$Rd), (ins PPRAny:$Pg, pprty:$Pn), 903 asm, "\t$Rd, $Pg, $Pn", 904 "", 905 []>, Sched<[]> { 906 bits<4> Pg; 907 bits<4> Pn; 908 bits<5> Rd; 909 let Inst{31-24} = 0b00100101; 910 let Inst{23-22} = sz8_64; 911 let Inst{21-19} = 0b100; 912 let Inst{18-16} = opc{3-1}; 913 let Inst{15-14} = 0b10; 914 let Inst{13-10} = Pg; 915 let Inst{9} = opc{0}; 916 let Inst{8-5} = Pn; 917 let Inst{4-0} = Rd; 918} 919 920multiclass sve_int_pcount_pred<bits<4> opc, string asm, 921 SDPatternOperator int_op> { 922 def _B : sve_int_pcount_pred<0b00, opc, asm, PPR8>; 923 def _H : sve_int_pcount_pred<0b01, opc, asm, PPR16>; 924 def _S : sve_int_pcount_pred<0b10, opc, asm, PPR32>; 925 def _D : sve_int_pcount_pred<0b11, opc, asm, PPR64>; 926 927 def : SVE_2_Op_Pat<i64, int_op, nxv16i1, nxv16i1, !cast<Instruction>(NAME # _B)>; 928 def : SVE_2_Op_Pat<i64, int_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 929 def : SVE_2_Op_Pat<i64, int_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 930 def : SVE_2_Op_Pat<i64, int_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 931} 932 933//===----------------------------------------------------------------------===// 934// SVE Element Count Group 935//===----------------------------------------------------------------------===// 936 937class sve_int_count<bits<3> opc, string asm> 938: I<(outs GPR64:$Rd), (ins sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 939 asm, "\t$Rd, $pattern, mul $imm4", 940 "", 941 []>, Sched<[]> { 942 bits<5> Rd; 943 bits<4> imm4; 944 bits<5> pattern; 945 let Inst{31-24} = 0b00000100; 946 let Inst{23-22} = opc{2-1}; 947 let Inst{21-20} = 0b10; 948 let Inst{19-16} = imm4; 949 let Inst{15-11} = 0b11100; 950 let Inst{10} = opc{0}; 951 let Inst{9-5} = pattern; 952 let Inst{4-0} = Rd; 953 954 let isReMaterializable = 1; 955} 956 957multiclass sve_int_count<bits<3> opc, string asm, SDPatternOperator op> { 958 def NAME : sve_int_count<opc, asm>; 959 960 def : InstAlias<asm # "\t$Rd, $pattern", 961 (!cast<Instruction>(NAME) GPR64:$Rd, sve_pred_enum:$pattern, 1), 1>; 962 def : InstAlias<asm # "\t$Rd", 963 (!cast<Instruction>(NAME) GPR64:$Rd, 0b11111, 1), 2>; 964 965 def : Pat<(i64 (mul (op sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm))), 966 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 967 968 def : Pat<(i64 (shl (op sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm))), 969 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, sve_incdec_imm:$imm)>; 970 971 def : Pat<(i64 (op sve_pred_enum:$pattern)), 972 (!cast<Instruction>(NAME) sve_pred_enum:$pattern, 1)>; 973} 974 975class sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty> 976: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 977 asm, "\t$Zdn, $pattern, mul $imm4", 978 "", 979 []>, Sched<[]> { 980 bits<5> Zdn; 981 bits<5> pattern; 982 bits<4> imm4; 983 let Inst{31-24} = 0b00000100; 984 let Inst{23-22} = opc{4-3}; 985 let Inst{21} = 0b1; 986 let Inst{20} = opc{2}; 987 let Inst{19-16} = imm4; 988 let Inst{15-12} = 0b1100; 989 let Inst{11-10} = opc{1-0}; 990 let Inst{9-5} = pattern; 991 let Inst{4-0} = Zdn; 992 993 let Constraints = "$Zdn = $_Zdn"; 994 let DestructiveInstType = DestructiveOther; 995 let ElementSize = ElementSizeNone; 996} 997 998multiclass sve_int_countvlv<bits<5> opc, string asm, ZPRRegOp zprty, 999 SDPatternOperator op = null_frag, 1000 ValueType vt = OtherVT> { 1001 def NAME : sve_int_countvlv<opc, asm, zprty>; 1002 1003 def : InstAlias<asm # "\t$Zdn, $pattern", 1004 (!cast<Instruction>(NAME) zprty:$Zdn, sve_pred_enum:$pattern, 1), 1>; 1005 def : InstAlias<asm # "\t$Zdn", 1006 (!cast<Instruction>(NAME) zprty:$Zdn, 0b11111, 1), 2>; 1007 1008 def : Pat<(vt (op (vt zprty:$Zn), (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1009 (!cast<Instruction>(NAME) $Zn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1010} 1011 1012class sve_int_pred_pattern_a<bits<3> opc, string asm> 1013: I<(outs GPR64:$Rdn), (ins GPR64:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 1014 asm, "\t$Rdn, $pattern, mul $imm4", 1015 "", 1016 []>, Sched<[]> { 1017 bits<5> Rdn; 1018 bits<5> pattern; 1019 bits<4> imm4; 1020 let Inst{31-24} = 0b00000100; 1021 let Inst{23-22} = opc{2-1}; 1022 let Inst{21-20} = 0b11; 1023 let Inst{19-16} = imm4; 1024 let Inst{15-11} = 0b11100; 1025 let Inst{10} = opc{0}; 1026 let Inst{9-5} = pattern; 1027 let Inst{4-0} = Rdn; 1028 1029 let Constraints = "$Rdn = $_Rdn"; 1030} 1031 1032multiclass sve_int_pred_pattern_a<bits<3> opc, string asm, 1033 SDPatternOperator op, 1034 SDPatternOperator opcnt> { 1035 let Predicates = [HasSVEorSME] in { 1036 def NAME : sve_int_pred_pattern_a<opc, asm>; 1037 1038 def : InstAlias<asm # "\t$Rdn, $pattern", 1039 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1040 def : InstAlias<asm # "\t$Rdn", 1041 (!cast<Instruction>(NAME) GPR64:$Rdn, 0b11111, 1), 2>; 1042 } 1043 1044 let Predicates = [HasSVEorSME, UseScalarIncVL] in { 1045 def : Pat<(i64 (op GPR64:$Rdn, (opcnt sve_pred_enum:$pattern))), 1046 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, 1)>; 1047 1048 def : Pat<(i64 (op GPR64:$Rdn, (mul (opcnt sve_pred_enum:$pattern), (sve_cnt_mul_imm_i64 i32:$imm)))), 1049 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1050 1051 def : Pat<(i64 (op GPR64:$Rdn, (shl (opcnt sve_pred_enum:$pattern), (sve_cnt_shl_imm i32:$imm)))), 1052 (!cast<Instruction>(NAME) GPR64:$Rdn, sve_pred_enum:$pattern, $imm)>; 1053 1054 def : Pat<(i32 (op GPR32:$Rdn, (i32 (trunc (opcnt (sve_pred_enum:$pattern)))))), 1055 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1056 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, 1), 1057 sub_32))>; 1058 1059 def : Pat<(i32 (op GPR32:$Rdn, (mul (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_mul_imm_i32 i32:$imm)))), 1060 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1061 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1062 sub_32))>; 1063 1064 def : Pat<(i32 (op GPR32:$Rdn, (shl (i32 (trunc (opcnt (sve_pred_enum:$pattern)))), (sve_cnt_shl_imm i32:$imm)))), 1065 (i32 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (i64 (IMPLICIT_DEF)), 1066 GPR32:$Rdn, sub_32), sve_pred_enum:$pattern, $imm), 1067 sub_32))>; 1068 } 1069} 1070 1071class sve_int_pred_pattern_b<bits<5> opc, string asm, RegisterOperand dt, 1072 RegisterOperand st> 1073: I<(outs dt:$Rdn), (ins st:$_Rdn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4), 1074 asm, "\t$Rdn, $pattern, mul $imm4", 1075 "", 1076 []>, Sched<[]> { 1077 bits<5> Rdn; 1078 bits<5> pattern; 1079 bits<4> imm4; 1080 let Inst{31-24} = 0b00000100; 1081 let Inst{23-22} = opc{4-3}; 1082 let Inst{21} = 0b1; 1083 let Inst{20} = opc{2}; 1084 let Inst{19-16} = imm4; 1085 let Inst{15-12} = 0b1111; 1086 let Inst{11-10} = opc{1-0}; 1087 let Inst{9-5} = pattern; 1088 let Inst{4-0} = Rdn; 1089 1090 // Signed 32bit forms require their GPR operand printed. 1091 let AsmString = !if(!eq(opc{2,0}, 0b00), 1092 !strconcat(asm, "\t$Rdn, $_Rdn, $pattern, mul $imm4"), 1093 !strconcat(asm, "\t$Rdn, $pattern, mul $imm4")); 1094 1095 let Constraints = "$Rdn = $_Rdn"; 1096} 1097 1098multiclass sve_int_pred_pattern_b_s32<bits<5> opc, string asm, 1099 SDPatternOperator op> { 1100 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64as32>; 1101 1102 def : InstAlias<asm # "\t$Rd, $Rn, $pattern", 1103 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, sve_pred_enum:$pattern, 1), 1>; 1104 def : InstAlias<asm # "\t$Rd, $Rn", 1105 (!cast<Instruction>(NAME) GPR64z:$Rd, GPR64as32:$Rn, 0b11111, 1), 2>; 1106 1107 // NOTE: Register allocation doesn't like tied operands of differing register 1108 // class, hence the extra INSERT_SUBREG complication. 1109 1110 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1111 (EXTRACT_SUBREG (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4), sub_32)>; 1112 def : Pat<(i64 (sext (i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))))), 1113 (!cast<Instruction>(NAME) (INSERT_SUBREG (IMPLICIT_DEF), $Rn, sub_32), sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1114} 1115 1116multiclass sve_int_pred_pattern_b_u32<bits<5> opc, string asm, 1117 SDPatternOperator op> { 1118 def NAME : sve_int_pred_pattern_b<opc, asm, GPR32z, GPR32z>; 1119 1120 def : InstAlias<asm # "\t$Rdn, $pattern", 1121 (!cast<Instruction>(NAME) GPR32z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1122 def : InstAlias<asm # "\t$Rdn", 1123 (!cast<Instruction>(NAME) GPR32z:$Rdn, 0b11111, 1), 2>; 1124 1125 def : Pat<(i32 (op GPR32:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1126 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1127} 1128 1129multiclass sve_int_pred_pattern_b_x64<bits<5> opc, string asm, 1130 SDPatternOperator op> { 1131 def NAME : sve_int_pred_pattern_b<opc, asm, GPR64z, GPR64z>; 1132 1133 def : InstAlias<asm # "\t$Rdn, $pattern", 1134 (!cast<Instruction>(NAME) GPR64z:$Rdn, sve_pred_enum:$pattern, 1), 1>; 1135 def : InstAlias<asm # "\t$Rdn", 1136 (!cast<Instruction>(NAME) GPR64z:$Rdn, 0b11111, 1), 2>; 1137 1138 def : Pat<(i64 (op GPR64:$Rn, (sve_pred_enum:$pattern), (sve_incdec_imm:$imm4))), 1139 (!cast<Instruction>(NAME) $Rn, sve_pred_enum:$pattern, sve_incdec_imm:$imm4)>; 1140} 1141 1142 1143//===----------------------------------------------------------------------===// 1144// SVE Permute - Cross Lane Group 1145//===----------------------------------------------------------------------===// 1146 1147class sve_int_perm_dup_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1148 ValueType vt, RegisterClass srcRegType, 1149 SDPatternOperator op> 1150: I<(outs zprty:$Zd), (ins srcRegType:$Rn), 1151 asm, "\t$Zd, $Rn", 1152 "", 1153 [(set (vt zprty:$Zd), (op srcRegType:$Rn))]>, Sched<[]> { 1154 bits<5> Rn; 1155 bits<5> Zd; 1156 let Inst{31-24} = 0b00000101; 1157 let Inst{23-22} = sz8_64; 1158 let Inst{21-10} = 0b100000001110; 1159 let Inst{9-5} = Rn; 1160 let Inst{4-0} = Zd; 1161} 1162 1163multiclass sve_int_perm_dup_r<string asm, SDPatternOperator op> { 1164 def _B : sve_int_perm_dup_r<0b00, asm, ZPR8, nxv16i8, GPR32sp, op>; 1165 def _H : sve_int_perm_dup_r<0b01, asm, ZPR16, nxv8i16, GPR32sp, op>; 1166 def _S : sve_int_perm_dup_r<0b10, asm, ZPR32, nxv4i32, GPR32sp, op>; 1167 def _D : sve_int_perm_dup_r<0b11, asm, ZPR64, nxv2i64, GPR64sp, op>; 1168 1169 def : InstAlias<"mov $Zd, $Rn", 1170 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, GPR32sp:$Rn), 1>; 1171 def : InstAlias<"mov $Zd, $Rn", 1172 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, GPR32sp:$Rn), 1>; 1173 def : InstAlias<"mov $Zd, $Rn", 1174 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, GPR32sp:$Rn), 1>; 1175 def : InstAlias<"mov $Zd, $Rn", 1176 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, GPR64sp:$Rn), 1>; 1177} 1178 1179class sve_int_perm_dup_i<bits<5> tsz, Operand immtype, string asm, 1180 ZPRRegOp zprty> 1181: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$idx), 1182 asm, "\t$Zd, $Zn$idx", 1183 "", 1184 []>, Sched<[]> { 1185 bits<5> Zd; 1186 bits<5> Zn; 1187 bits<7> idx; 1188 let Inst{31-24} = 0b00000101; 1189 let Inst{23-22} = {?,?}; // imm3h 1190 let Inst{21} = 0b1; 1191 let Inst{20-16} = tsz; 1192 let Inst{15-10} = 0b001000; 1193 let Inst{9-5} = Zn; 1194 let Inst{4-0} = Zd; 1195} 1196 1197multiclass sve_int_perm_dup_i<string asm> { 1198 def _B : sve_int_perm_dup_i<{?,?,?,?,1}, sve_elm_idx_extdup_b, asm, ZPR8> { 1199 let Inst{23-22} = idx{5-4}; 1200 let Inst{20-17} = idx{3-0}; 1201 } 1202 def _H : sve_int_perm_dup_i<{?,?,?,1,0}, sve_elm_idx_extdup_h, asm, ZPR16> { 1203 let Inst{23-22} = idx{4-3}; 1204 let Inst{20-18} = idx{2-0}; 1205 } 1206 def _S : sve_int_perm_dup_i<{?,?,1,0,0}, sve_elm_idx_extdup_s, asm, ZPR32> { 1207 let Inst{23-22} = idx{3-2}; 1208 let Inst{20-19} = idx{1-0}; 1209 } 1210 def _D : sve_int_perm_dup_i<{?,1,0,0,0}, sve_elm_idx_extdup_d, asm, ZPR64> { 1211 let Inst{23-22} = idx{2-1}; 1212 let Inst{20} = idx{0}; 1213 } 1214 def _Q : sve_int_perm_dup_i<{1,0,0,0,0}, sve_elm_idx_extdup_q, asm, ZPR128> { 1215 let Inst{23-22} = idx{1-0}; 1216 } 1217 1218 def : InstAlias<"mov $Zd, $Zn$idx", 1219 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, sve_elm_idx_extdup_b:$idx), 1>; 1220 def : InstAlias<"mov $Zd, $Zn$idx", 1221 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, sve_elm_idx_extdup_h:$idx), 1>; 1222 def : InstAlias<"mov $Zd, $Zn$idx", 1223 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, sve_elm_idx_extdup_s:$idx), 1>; 1224 def : InstAlias<"mov $Zd, $Zn$idx", 1225 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, sve_elm_idx_extdup_d:$idx), 1>; 1226 def : InstAlias<"mov $Zd, $Zn$idx", 1227 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, ZPR128:$Zn, sve_elm_idx_extdup_q:$idx), 1>; 1228 def : InstAlias<"mov $Zd, $Bn", 1229 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, FPR8asZPR:$Bn, 0), 2>; 1230 def : InstAlias<"mov $Zd, $Hn", 1231 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, FPR16asZPR:$Hn, 0), 2>; 1232 def : InstAlias<"mov $Zd, $Sn", 1233 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, FPR32asZPR:$Sn, 0), 2>; 1234 def : InstAlias<"mov $Zd, $Dn", 1235 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, FPR64asZPR:$Dn, 0), 2>; 1236 def : InstAlias<"mov $Zd, $Qn", 1237 (!cast<Instruction>(NAME # _Q) ZPR128:$Zd, FPR128asZPR:$Qn, 0), 2>; 1238 1239 // Duplicate extracted element of vector into all vector elements 1240 def : Pat<(nxv16i8 (splat_vector (i32 (vector_extract (nxv16i8 ZPR:$vec), sve_elm_idx_extdup_b:$index)))), 1241 (!cast<Instruction>(NAME # _B) ZPR:$vec, sve_elm_idx_extdup_b:$index)>; 1242 def : Pat<(nxv8i16 (splat_vector (i32 (vector_extract (nxv8i16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1243 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1244 def : Pat<(nxv4i32 (splat_vector (i32 (vector_extract (nxv4i32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1245 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1246 def : Pat<(nxv2i64 (splat_vector (i64 (vector_extract (nxv2i64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1247 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1248 def : Pat<(nxv8f16 (splat_vector (f16 (vector_extract (nxv8f16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1249 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1250 def : Pat<(nxv8bf16 (splat_vector (bf16 (vector_extract (nxv8bf16 ZPR:$vec), sve_elm_idx_extdup_h:$index)))), 1251 (!cast<Instruction>(NAME # _H) ZPR:$vec, sve_elm_idx_extdup_h:$index)>; 1252 def : Pat<(nxv4f16 (splat_vector (f16 (vector_extract (nxv4f16 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1253 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1254 def : Pat<(nxv2f16 (splat_vector (f16 (vector_extract (nxv2f16 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1255 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1256 def : Pat<(nxv4f32 (splat_vector (f32 (vector_extract (nxv4f32 ZPR:$vec), sve_elm_idx_extdup_s:$index)))), 1257 (!cast<Instruction>(NAME # _S) ZPR:$vec, sve_elm_idx_extdup_s:$index)>; 1258 def : Pat<(nxv2f32 (splat_vector (f32 (vector_extract (nxv2f32 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1259 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1260 def : Pat<(nxv2f64 (splat_vector (f64 (vector_extract (nxv2f64 ZPR:$vec), sve_elm_idx_extdup_d:$index)))), 1261 (!cast<Instruction>(NAME # _D) ZPR:$vec, sve_elm_idx_extdup_d:$index)>; 1262 1263 def : Pat<(nxv16i8 (AArch64duplane128 nxv16i8:$Op1, i64:$imm)), 1264 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1265 def : Pat<(nxv8i16 (AArch64duplane128 nxv8i16:$Op1, i64:$imm)), 1266 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1267 def : Pat<(nxv4i32 (AArch64duplane128 nxv4i32:$Op1, i64:$imm)), 1268 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1269 def : Pat<(nxv2i64 (AArch64duplane128 nxv2i64:$Op1, i64:$imm)), 1270 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1271 def : Pat<(nxv8f16 (AArch64duplane128 nxv8f16:$Op1, i64:$imm)), 1272 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1273 def : Pat<(nxv4f32 (AArch64duplane128 nxv4f32:$Op1, i64:$imm)), 1274 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1275 def : Pat<(nxv2f64 (AArch64duplane128 nxv2f64:$Op1, i64:$imm)), 1276 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1277 def : Pat<(nxv8bf16 (AArch64duplane128 nxv8bf16:$Op1, i64:$imm)), 1278 (!cast<Instruction>(NAME # _Q) $Op1, $imm)>; 1279} 1280 1281class sve_int_perm_tbl<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty, 1282 RegisterOperand VecList> 1283: I<(outs zprty:$Zd), (ins VecList:$Zn, zprty:$Zm), 1284 asm, "\t$Zd, $Zn, $Zm", 1285 "", 1286 []>, Sched<[]> { 1287 bits<5> Zd; 1288 bits<5> Zm; 1289 bits<5> Zn; 1290 let Inst{31-24} = 0b00000101; 1291 let Inst{23-22} = sz8_64; 1292 let Inst{21} = 0b1; 1293 let Inst{20-16} = Zm; 1294 let Inst{15-13} = 0b001; 1295 let Inst{12-11} = opc; 1296 let Inst{10} = 0b0; 1297 let Inst{9-5} = Zn; 1298 let Inst{4-0} = Zd; 1299} 1300 1301multiclass sve_int_perm_tbl<string asm, SDPatternOperator op> { 1302 def _B : sve_int_perm_tbl<0b00, 0b10, asm, ZPR8, Z_b>; 1303 def _H : sve_int_perm_tbl<0b01, 0b10, asm, ZPR16, Z_h>; 1304 def _S : sve_int_perm_tbl<0b10, 0b10, asm, ZPR32, Z_s>; 1305 def _D : sve_int_perm_tbl<0b11, 0b10, asm, ZPR64, Z_d>; 1306 1307 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1308 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 0>; 1309 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1310 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 0>; 1311 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1312 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 0>; 1313 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 1314 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, ZPR64:$Zn, ZPR64:$Zm), 0>; 1315 1316 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1317 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1318 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1319 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1320 1321 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1322 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1323 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1324 1325 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1326} 1327 1328multiclass sve2_int_perm_tbl<string asm, SDPatternOperator op> { 1329 def _B : sve_int_perm_tbl<0b00, 0b01, asm, ZPR8, ZZ_b>; 1330 def _H : sve_int_perm_tbl<0b01, 0b01, asm, ZPR16, ZZ_h>; 1331 def _S : sve_int_perm_tbl<0b10, 0b01, asm, ZPR32, ZZ_s>; 1332 def _D : sve_int_perm_tbl<0b11, 0b01, asm, ZPR64, ZZ_d>; 1333 1334 def : Pat<(nxv16i8 (op nxv16i8:$Op1, nxv16i8:$Op2, nxv16i8:$Op3)), 1335 (nxv16i8 (!cast<Instruction>(NAME # _B) (REG_SEQUENCE ZPR2, nxv16i8:$Op1, zsub0, 1336 nxv16i8:$Op2, zsub1), 1337 nxv16i8:$Op3))>; 1338 1339 def : Pat<(nxv8i16 (op nxv8i16:$Op1, nxv8i16:$Op2, nxv8i16:$Op3)), 1340 (nxv8i16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8i16:$Op1, zsub0, 1341 nxv8i16:$Op2, zsub1), 1342 nxv8i16:$Op3))>; 1343 1344 def : Pat<(nxv4i32 (op nxv4i32:$Op1, nxv4i32:$Op2, nxv4i32:$Op3)), 1345 (nxv4i32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4i32:$Op1, zsub0, 1346 nxv4i32:$Op2, zsub1), 1347 nxv4i32:$Op3))>; 1348 1349 def : Pat<(nxv2i64 (op nxv2i64:$Op1, nxv2i64:$Op2, nxv2i64:$Op3)), 1350 (nxv2i64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2i64:$Op1, zsub0, 1351 nxv2i64:$Op2, zsub1), 1352 nxv2i64:$Op3))>; 1353 1354 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8i16:$Op3)), 1355 (nxv8f16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8f16:$Op1, zsub0, 1356 nxv8f16:$Op2, zsub1), 1357 nxv8i16:$Op3))>; 1358 1359 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4i32:$Op3)), 1360 (nxv4f32 (!cast<Instruction>(NAME # _S) (REG_SEQUENCE ZPR2, nxv4f32:$Op1, zsub0, 1361 nxv4f32:$Op2, zsub1), 1362 nxv4i32:$Op3))>; 1363 1364 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2i64:$Op3)), 1365 (nxv2f64 (!cast<Instruction>(NAME # _D) (REG_SEQUENCE ZPR2, nxv2f64:$Op1, zsub0, 1366 nxv2f64:$Op2, zsub1), 1367 nxv2i64:$Op3))>; 1368 1369 def : Pat<(nxv8bf16 (op nxv8bf16:$Op1, nxv8bf16:$Op2, nxv8i16:$Op3)), 1370 (nxv8bf16 (!cast<Instruction>(NAME # _H) (REG_SEQUENCE ZPR2, nxv8bf16:$Op1, zsub0, 1371 nxv8bf16:$Op2, zsub1), 1372 nxv8i16:$Op3))>; 1373} 1374 1375class sve2_int_perm_tbx<bits<2> sz8_64, bits<2> opc, string asm, ZPRRegOp zprty> 1376: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, zprty:$Zm), 1377 asm, "\t$Zd, $Zn, $Zm", 1378 "", 1379 []>, Sched<[]> { 1380 bits<5> Zd; 1381 bits<5> Zm; 1382 bits<5> Zn; 1383 let Inst{31-24} = 0b00000101; 1384 let Inst{23-22} = sz8_64; 1385 let Inst{21} = 0b1; 1386 let Inst{20-16} = Zm; 1387 let Inst{15-13} = 0b001; 1388 let Inst{12-11} = opc; 1389 let Inst{10} = 0b1; 1390 let Inst{9-5} = Zn; 1391 let Inst{4-0} = Zd; 1392 1393 let Constraints = "$Zd = $_Zd"; 1394} 1395 1396multiclass sve2_int_perm_tbx<string asm, bits<2> opc, SDPatternOperator op> { 1397 def _B : sve2_int_perm_tbx<0b00, opc, asm, ZPR8>; 1398 def _H : sve2_int_perm_tbx<0b01, opc, asm, ZPR16>; 1399 def _S : sve2_int_perm_tbx<0b10, opc, asm, ZPR32>; 1400 def _D : sve2_int_perm_tbx<0b11, opc, asm, ZPR64>; 1401 1402 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1403 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1404 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1405 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1406 1407 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1408 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1409 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1410 1411 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1412} 1413 1414class sve_int_perm_reverse_z<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1415: I<(outs zprty:$Zd), (ins zprty:$Zn), 1416 asm, "\t$Zd, $Zn", 1417 "", 1418 []>, Sched<[]> { 1419 bits<5> Zd; 1420 bits<5> Zn; 1421 let Inst{31-24} = 0b00000101; 1422 let Inst{23-22} = sz8_64; 1423 let Inst{21-10} = 0b111000001110; 1424 let Inst{9-5} = Zn; 1425 let Inst{4-0} = Zd; 1426} 1427 1428multiclass sve_int_perm_reverse_z<string asm, SDPatternOperator op> { 1429 def _B : sve_int_perm_reverse_z<0b00, asm, ZPR8>; 1430 def _H : sve_int_perm_reverse_z<0b01, asm, ZPR16>; 1431 def _S : sve_int_perm_reverse_z<0b10, asm, ZPR32>; 1432 def _D : sve_int_perm_reverse_z<0b11, asm, ZPR64>; 1433 1434 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME # _B)>; 1435 def : SVE_1_Op_Pat<nxv8i16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 1436 def : SVE_1_Op_Pat<nxv4i32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 1437 def : SVE_1_Op_Pat<nxv2i64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 1438 1439 def : SVE_1_Op_Pat<nxv2f16, op, nxv2f16, !cast<Instruction>(NAME # _D)>; 1440 def : SVE_1_Op_Pat<nxv4f16, op, nxv4f16, !cast<Instruction>(NAME # _S)>; 1441 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 1442 def : SVE_1_Op_Pat<nxv2f32, op, nxv2f32, !cast<Instruction>(NAME # _D)>; 1443 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 1444 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 1445 1446 def : SVE_1_Op_Pat<nxv2bf16, op, nxv2bf16, !cast<Instruction>(NAME # _D)>; 1447 def : SVE_1_Op_Pat<nxv4bf16, op, nxv4bf16, !cast<Instruction>(NAME # _S)>; 1448 def : SVE_1_Op_Pat<nxv8bf16, op, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1449} 1450 1451class sve_int_perm_reverse_p<bits<2> sz8_64, string asm, PPRRegOp pprty, 1452 SDPatternOperator op> 1453: I<(outs pprty:$Pd), (ins pprty:$Pn), 1454 asm, "\t$Pd, $Pn", 1455 "", 1456 [(set nxv16i1:$Pd, (op nxv16i1:$Pn))]>, Sched<[]> { 1457 bits<4> Pd; 1458 bits<4> Pn; 1459 let Inst{31-24} = 0b00000101; 1460 let Inst{23-22} = sz8_64; 1461 let Inst{21-9} = 0b1101000100000; 1462 let Inst{8-5} = Pn; 1463 let Inst{4} = 0b0; 1464 let Inst{3-0} = Pd; 1465} 1466 1467multiclass sve_int_perm_reverse_p<string asm, SDPatternOperator ir_op, 1468 SDPatternOperator op_b16, 1469 SDPatternOperator op_b32, 1470 SDPatternOperator op_b64> { 1471 def _B : sve_int_perm_reverse_p<0b00, asm, PPR8, ir_op>; 1472 def _H : sve_int_perm_reverse_p<0b01, asm, PPR16, op_b16>; 1473 def _S : sve_int_perm_reverse_p<0b10, asm, PPR32, op_b32>; 1474 def _D : sve_int_perm_reverse_p<0b11, asm, PPR64, op_b64>; 1475 1476 def : SVE_1_Op_Pat<nxv8i1, ir_op, nxv8i1, !cast<Instruction>(NAME # _H)>; 1477 def : SVE_1_Op_Pat<nxv4i1, ir_op, nxv4i1, !cast<Instruction>(NAME # _S)>; 1478 def : SVE_1_Op_Pat<nxv2i1, ir_op, nxv2i1, !cast<Instruction>(NAME # _D)>; 1479} 1480 1481class sve_int_perm_unpk<bits<2> sz16_64, bits<2> opc, string asm, 1482 ZPRRegOp zprty1, ZPRRegOp zprty2> 1483: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 1484 asm, "\t$Zd, $Zn", 1485 "", []>, Sched<[]> { 1486 bits<5> Zd; 1487 bits<5> Zn; 1488 let Inst{31-24} = 0b00000101; 1489 let Inst{23-22} = sz16_64; 1490 let Inst{21-18} = 0b1100; 1491 let Inst{17-16} = opc; 1492 let Inst{15-10} = 0b001110; 1493 let Inst{9-5} = Zn; 1494 let Inst{4-0} = Zd; 1495} 1496 1497multiclass sve_int_perm_unpk<bits<2> opc, string asm, SDPatternOperator op> { 1498 def _H : sve_int_perm_unpk<0b01, opc, asm, ZPR16, ZPR8>; 1499 def _S : sve_int_perm_unpk<0b10, opc, asm, ZPR32, ZPR16>; 1500 def _D : sve_int_perm_unpk<0b11, opc, asm, ZPR64, ZPR32>; 1501 1502 def : SVE_1_Op_Pat<nxv8i16, op, nxv16i8, !cast<Instruction>(NAME # _H)>; 1503 def : SVE_1_Op_Pat<nxv4i32, op, nxv8i16, !cast<Instruction>(NAME # _S)>; 1504 def : SVE_1_Op_Pat<nxv2i64, op, nxv4i32, !cast<Instruction>(NAME # _D)>; 1505} 1506 1507class sve_int_perm_insrs<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1508 RegisterClass srcRegType> 1509: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcRegType:$Rm), 1510 asm, "\t$Zdn, $Rm", 1511 "", 1512 []>, Sched<[]> { 1513 bits<5> Rm; 1514 bits<5> Zdn; 1515 let Inst{31-24} = 0b00000101; 1516 let Inst{23-22} = sz8_64; 1517 let Inst{21-10} = 0b100100001110; 1518 let Inst{9-5} = Rm; 1519 let Inst{4-0} = Zdn; 1520 1521 let Constraints = "$Zdn = $_Zdn"; 1522 let DestructiveInstType = DestructiveOther; 1523} 1524 1525multiclass sve_int_perm_insrs<string asm, SDPatternOperator op> { 1526 def _B : sve_int_perm_insrs<0b00, asm, ZPR8, GPR32>; 1527 def _H : sve_int_perm_insrs<0b01, asm, ZPR16, GPR32>; 1528 def _S : sve_int_perm_insrs<0b10, asm, ZPR32, GPR32>; 1529 def _D : sve_int_perm_insrs<0b11, asm, ZPR64, GPR64>; 1530 1531 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 1532 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 1533 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 1534 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, i64, !cast<Instruction>(NAME # _D)>; 1535} 1536 1537class sve_int_perm_insrv<bits<2> sz8_64, string asm, ZPRRegOp zprty, 1538 FPRasZPROperand srcOpType> 1539: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, srcOpType:$Vm), 1540 asm, "\t$Zdn, $Vm", 1541 "", 1542 []>, Sched<[]> { 1543 bits<5> Vm; 1544 bits<5> Zdn; 1545 let Inst{31-24} = 0b00000101; 1546 let Inst{23-22} = sz8_64; 1547 let Inst{21-10} = 0b110100001110; 1548 let Inst{9-5} = Vm; 1549 let Inst{4-0} = Zdn; 1550 1551 let Constraints = "$Zdn = $_Zdn"; 1552 let DestructiveInstType = DestructiveOther; 1553} 1554 1555multiclass sve_int_perm_insrv<string asm, SDPatternOperator op> { 1556 def _B : sve_int_perm_insrv<0b00, asm, ZPR8, FPR8asZPR>; 1557 def _H : sve_int_perm_insrv<0b01, asm, ZPR16, FPR16asZPR>; 1558 def _S : sve_int_perm_insrv<0b10, asm, ZPR32, FPR32asZPR>; 1559 def _D : sve_int_perm_insrv<0b11, asm, ZPR64, FPR64asZPR>; 1560 1561 def : Pat<(nxv8f16 (op nxv8f16:$Zn, f16:$Vm)), 1562 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1563 def : Pat<(nxv4f32 (op nxv4f32:$Zn, f32:$Vm)), 1564 (!cast<Instruction>(NAME # _S) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, ssub))>; 1565 def : Pat<(nxv2f64 (op nxv2f64:$Zn, f64:$Vm)), 1566 (!cast<Instruction>(NAME # _D) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, dsub))>; 1567 1568 def : Pat<(nxv8bf16 (op nxv8bf16:$Zn, bf16:$Vm)), 1569 (!cast<Instruction>(NAME # _H) $Zn, (INSERT_SUBREG (IMPLICIT_DEF), $Vm, hsub))>; 1570 1571 // Keep integer insertions within the vector unit. 1572 def : Pat<(nxv16i8 (op (nxv16i8 ZPR:$Zn), (i32 (vector_extract (nxv16i8 ZPR:$Vm), 0)))), 1573 (!cast<Instruction>(NAME # _B) $Zn, ZPR:$Vm)>; 1574 def : Pat<(nxv8i16 (op (nxv8i16 ZPR:$Zn), (i32 (vector_extract (nxv8i16 ZPR:$Vm), 0)))), 1575 (!cast<Instruction>(NAME # _H) $Zn, ZPR:$Vm)>; 1576 def : Pat<(nxv4i32 (op (nxv4i32 ZPR:$Zn), (i32 (vector_extract (nxv4i32 ZPR:$Vm), 0)))), 1577 (!cast<Instruction>(NAME # _S) $Zn, ZPR: $Vm)>; 1578 def : Pat<(nxv2i64 (op (nxv2i64 ZPR:$Zn), (i64 (vector_extract (nxv2i64 ZPR:$Vm), 0)))), 1579 (!cast<Instruction>(NAME # _D) $Zn, ZPR:$Vm)>; 1580 1581} 1582 1583//===----------------------------------------------------------------------===// 1584// SVE Permute - Extract Group 1585//===----------------------------------------------------------------------===// 1586 1587class sve_int_perm_extract_i<string asm> 1588: I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_255:$imm8), 1589 asm, "\t$Zdn, $_Zdn, $Zm, $imm8", 1590 "", []>, Sched<[]> { 1591 bits<5> Zdn; 1592 bits<5> Zm; 1593 bits<8> imm8; 1594 let Inst{31-21} = 0b00000101001; 1595 let Inst{20-16} = imm8{7-3}; 1596 let Inst{15-13} = 0b000; 1597 let Inst{12-10} = imm8{2-0}; 1598 let Inst{9-5} = Zm; 1599 let Inst{4-0} = Zdn; 1600 1601 let Constraints = "$Zdn = $_Zdn"; 1602 let DestructiveInstType = DestructiveOther; 1603 let ElementSize = ElementSizeNone; 1604} 1605 1606multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> { 1607 def NAME : sve_int_perm_extract_i<asm>; 1608 1609 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255, 1610 !cast<Instruction>(NAME)>; 1611} 1612 1613class sve2_int_perm_extract_i_cons<string asm> 1614: I<(outs ZPR8:$Zd), (ins ZZ_b:$Zn, imm0_255:$imm8), 1615 asm, "\t$Zd, $Zn, $imm8", 1616 "", []>, Sched<[]> { 1617 bits<5> Zd; 1618 bits<5> Zn; 1619 bits<8> imm8; 1620 let Inst{31-21} = 0b00000101011; 1621 let Inst{20-16} = imm8{7-3}; 1622 let Inst{15-13} = 0b000; 1623 let Inst{12-10} = imm8{2-0}; 1624 let Inst{9-5} = Zn; 1625 let Inst{4-0} = Zd; 1626} 1627 1628//===----------------------------------------------------------------------===// 1629// SVE Vector Select Group 1630//===----------------------------------------------------------------------===// 1631 1632class sve_int_sel_vvv<bits<2> sz8_64, string asm, ZPRRegOp zprty> 1633: I<(outs zprty:$Zd), (ins PPRAny:$Pg, zprty:$Zn, zprty:$Zm), 1634 asm, "\t$Zd, $Pg, $Zn, $Zm", 1635 "", 1636 []>, Sched<[]> { 1637 bits<4> Pg; 1638 bits<5> Zd; 1639 bits<5> Zm; 1640 bits<5> Zn; 1641 let Inst{31-24} = 0b00000101; 1642 let Inst{23-22} = sz8_64; 1643 let Inst{21} = 0b1; 1644 let Inst{20-16} = Zm; 1645 let Inst{15-14} = 0b11; 1646 let Inst{13-10} = Pg; 1647 let Inst{9-5} = Zn; 1648 let Inst{4-0} = Zd; 1649} 1650 1651multiclass sve_int_sel_vvv<string asm, SDPatternOperator op> { 1652 def _B : sve_int_sel_vvv<0b00, asm, ZPR8>; 1653 def _H : sve_int_sel_vvv<0b01, asm, ZPR16>; 1654 def _S : sve_int_sel_vvv<0b10, asm, ZPR32>; 1655 def _D : sve_int_sel_vvv<0b11, asm, ZPR64>; 1656 1657 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1658 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1659 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1660 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1661 1662 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1663 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 1664 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1665 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 1666 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 1667 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1668 1669 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 1670 1671 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1672 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPRAny:$Pg, ZPR8:$Zn, ZPR8:$Zd), 1>; 1673 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1674 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, ZPR16:$Zn, ZPR16:$Zd), 1>; 1675 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1676 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, ZPR32:$Zn, ZPR32:$Zd), 1>; 1677 def : InstAlias<"mov $Zd, $Pg/m, $Zn", 1678 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, ZPR64:$Zn, ZPR64:$Zd), 1>; 1679} 1680 1681 1682//===----------------------------------------------------------------------===// 1683// SVE Predicate Logical Operations Group 1684//===----------------------------------------------------------------------===// 1685 1686class sve_int_pred_log<bits<4> opc, string asm> 1687: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 1688 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 1689 "", 1690 []>, Sched<[]> { 1691 bits<4> Pd; 1692 bits<4> Pg; 1693 bits<4> Pm; 1694 bits<4> Pn; 1695 let Inst{31-24} = 0b00100101; 1696 let Inst{23-22} = opc{3-2}; 1697 let Inst{21-20} = 0b00; 1698 let Inst{19-16} = Pm; 1699 let Inst{15-14} = 0b01; 1700 let Inst{13-10} = Pg; 1701 let Inst{9} = opc{1}; 1702 let Inst{8-5} = Pn; 1703 let Inst{4} = opc{0}; 1704 let Inst{3-0} = Pd; 1705 1706 // SEL has no predication qualifier. 1707 let AsmString = !if(!eq(opc, 0b0011), 1708 !strconcat(asm, "\t$Pd, $Pg, $Pn, $Pm"), 1709 !strconcat(asm, "\t$Pd, $Pg/z, $Pn, $Pm")); 1710 1711 let Defs = !if(!eq (opc{2}, 1), [NZCV], []); 1712 1713} 1714 1715multiclass sve_int_pred_log<bits<4> opc, string asm, SDPatternOperator op, 1716 SDPatternOperator op_nopred = null_frag> { 1717 def NAME : sve_int_pred_log<opc, asm>; 1718 1719 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 1720 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i1, nxv8i1, !cast<Instruction>(NAME)>; 1721 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i1, nxv4i1, !cast<Instruction>(NAME)>; 1722 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2i1, nxv2i1, !cast<Instruction>(NAME)>; 1723 def : SVE_3_Op_Pat<nxv1i1, op, nxv1i1, nxv1i1, nxv1i1, !cast<Instruction>(NAME)>; 1724 def : SVE_2_Op_AllActive_Pat<nxv16i1, op_nopred, nxv16i1, nxv16i1, 1725 !cast<Instruction>(NAME), PTRUE_B>; 1726 def : SVE_2_Op_AllActive_Pat<nxv8i1, op_nopred, nxv8i1, nxv8i1, 1727 !cast<Instruction>(NAME), PTRUE_H>; 1728 def : SVE_2_Op_AllActive_Pat<nxv4i1, op_nopred, nxv4i1, nxv4i1, 1729 !cast<Instruction>(NAME), PTRUE_S>; 1730 def : SVE_2_Op_AllActive_Pat<nxv2i1, op_nopred, nxv2i1, nxv2i1, 1731 !cast<Instruction>(NAME), PTRUE_D>; 1732 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. 1733 def : SVE_2_Op_AllActive_Pat<nxv1i1, op_nopred, nxv1i1, nxv1i1, 1734 !cast<Instruction>(NAME), PTRUE_D>; 1735} 1736 1737// An instance of sve_int_pred_log_and but uses op_nopred's first operand as the 1738// general predicate. 1739multiclass sve_int_pred_log_v2<bits<4> opc, string asm, SDPatternOperator op, 1740 SDPatternOperator op_nopred> : 1741 sve_int_pred_log<opc, asm, op> { 1742 def : Pat<(nxv16i1 (op_nopred nxv16i1:$Op1, nxv16i1:$Op2)), 1743 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1744 def : Pat<(nxv8i1 (op_nopred nxv8i1:$Op1, nxv8i1:$Op2)), 1745 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1746 def : Pat<(nxv4i1 (op_nopred nxv4i1:$Op1, nxv4i1:$Op2)), 1747 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1748 def : Pat<(nxv2i1 (op_nopred nxv2i1:$Op1, nxv2i1:$Op2)), 1749 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1750 // Emulate .Q operation using a PTRUE_D when the other lanes don't matter. 1751 def : Pat<(nxv1i1 (op_nopred nxv1i1:$Op1, nxv1i1:$Op2)), 1752 (!cast<Instruction>(NAME) $Op1, $Op1, $Op2)>; 1753} 1754 1755//===----------------------------------------------------------------------===// 1756// SVE Logical Mask Immediate Group 1757//===----------------------------------------------------------------------===// 1758 1759class sve_int_log_imm<bits<2> opc, string asm> 1760: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, logical_imm64:$imms13), 1761 asm, "\t$Zdn, $_Zdn, $imms13", 1762 "", []>, Sched<[]> { 1763 bits<5> Zdn; 1764 bits<13> imms13; 1765 let Inst{31-24} = 0b00000101; 1766 let Inst{23-22} = opc; 1767 let Inst{21-18} = 0b0000; 1768 let Inst{17-5} = imms13; 1769 let Inst{4-0} = Zdn; 1770 1771 let Constraints = "$Zdn = $_Zdn"; 1772 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1773 let DestructiveInstType = DestructiveOther; 1774 let ElementSize = ElementSizeNone; 1775} 1776 1777multiclass sve_int_log_imm<bits<2> opc, string asm, string alias, SDPatternOperator op> { 1778 def NAME : sve_int_log_imm<opc, asm>; 1779 1780 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8Pat, !cast<Instruction>(NAME)>; 1781 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16Pat, !cast<Instruction>(NAME)>; 1782 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32Pat, !cast<Instruction>(NAME)>; 1783 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64Pat, !cast<Instruction>(NAME)>; 1784 1785 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1786 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8:$imm), 4>; 1787 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1788 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16:$imm), 3>; 1789 def : InstAlias<asm # "\t$Zdn, $Zdn, $imm", 1790 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32:$imm), 2>; 1791 1792 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1793 (!cast<Instruction>(NAME) ZPR8:$Zdn, sve_logical_imm8_not:$imm), 0>; 1794 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1795 (!cast<Instruction>(NAME) ZPR16:$Zdn, sve_logical_imm16_not:$imm), 0>; 1796 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1797 (!cast<Instruction>(NAME) ZPR32:$Zdn, sve_logical_imm32_not:$imm), 0>; 1798 def : InstAlias<alias # "\t$Zdn, $Zdn, $imm", 1799 (!cast<Instruction>(NAME) ZPR64:$Zdn, logical_imm64_not:$imm), 0>; 1800} 1801 1802multiclass sve_int_log_imm_bic<SDPatternOperator op> { 1803 def : SVE_1_Op_Imm_Log_Pat<nxv16i8, op, ZPR8, i32, SVELogicalImm8NotPat, !cast<Instruction>("AND_ZI")>; 1804 def : SVE_1_Op_Imm_Log_Pat<nxv8i16, op, ZPR16, i32, SVELogicalImm16NotPat, !cast<Instruction>("AND_ZI")>; 1805 def : SVE_1_Op_Imm_Log_Pat<nxv4i32, op, ZPR32, i32, SVELogicalImm32NotPat, !cast<Instruction>("AND_ZI")>; 1806 def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI")>; 1807} 1808 1809class sve_int_dup_mask_imm<string asm> 1810: I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), 1811 asm, "\t$Zd, $imms", 1812 "", 1813 []>, Sched<[]> { 1814 bits<5> Zd; 1815 bits<13> imms; 1816 let Inst{31-18} = 0b00000101110000; 1817 let Inst{17-5} = imms; 1818 let Inst{4-0} = Zd; 1819 1820 let isReMaterializable = 1; 1821 let DecoderMethod = "DecodeSVELogicalImmInstruction"; 1822} 1823 1824multiclass sve_int_dup_mask_imm<string asm> { 1825 def NAME : sve_int_dup_mask_imm<asm>; 1826 1827 def : InstAlias<"dupm $Zd, $imm", 1828 (!cast<Instruction>(NAME) ZPR8:$Zd, sve_logical_imm8:$imm), 4>; 1829 def : InstAlias<"dupm $Zd, $imm", 1830 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_logical_imm16:$imm), 3>; 1831 def : InstAlias<"dupm $Zd, $imm", 1832 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_logical_imm32:$imm), 2>; 1833 1834 // All Zd.b forms have a CPY/DUP equivalent, hence no byte alias here. 1835 def : InstAlias<"mov $Zd, $imm", 1836 (!cast<Instruction>(NAME) ZPR16:$Zd, sve_preferred_logical_imm16:$imm), 7>; 1837 def : InstAlias<"mov $Zd, $imm", 1838 (!cast<Instruction>(NAME) ZPR32:$Zd, sve_preferred_logical_imm32:$imm), 6>; 1839 def : InstAlias<"mov $Zd, $imm", 1840 (!cast<Instruction>(NAME) ZPR64:$Zd, sve_preferred_logical_imm64:$imm), 5>; 1841 1842 // NOTE: No pattern for nxv16i8 because DUP has full coverage. 1843 def : Pat<(nxv8i16 (splat_vector (i32 (SVELogicalImm16Pat i64:$imm)))), 1844 (!cast<Instruction>(NAME) i64:$imm)>; 1845 def : Pat<(nxv4i32 (splat_vector (i32 (SVELogicalImm32Pat i64:$imm)))), 1846 (!cast<Instruction>(NAME) i64:$imm)>; 1847 def : Pat<(nxv2i64 (splat_vector (i64 (SVELogicalImm64Pat i64:$imm)))), 1848 (!cast<Instruction>(NAME) i64:$imm)>; 1849} 1850 1851//===----------------------------------------------------------------------===// 1852// SVE Integer Arithmetic - Unpredicated Group. 1853//===----------------------------------------------------------------------===// 1854 1855class sve_int_bin_cons_arit_0<bits<2> sz8_64, bits<3> opc, string asm, 1856 ZPRRegOp zprty> 1857: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 1858 asm, "\t$Zd, $Zn, $Zm", 1859 "", []>, Sched<[]> { 1860 bits<5> Zd; 1861 bits<5> Zm; 1862 bits<5> Zn; 1863 let Inst{31-24} = 0b00000100; 1864 let Inst{23-22} = sz8_64; 1865 let Inst{21} = 0b1; 1866 let Inst{20-16} = Zm; 1867 let Inst{15-13} = 0b000; 1868 let Inst{12-10} = opc; 1869 let Inst{9-5} = Zn; 1870 let Inst{4-0} = Zd; 1871} 1872 1873multiclass sve_int_bin_cons_arit_0<bits<3> opc, string asm, SDPatternOperator op> { 1874 def _B : sve_int_bin_cons_arit_0<0b00, opc, asm, ZPR8>; 1875 def _H : sve_int_bin_cons_arit_0<0b01, opc, asm, ZPR16>; 1876 def _S : sve_int_bin_cons_arit_0<0b10, opc, asm, ZPR32>; 1877 def _D : sve_int_bin_cons_arit_0<0b11, opc, asm, ZPR64>; 1878 1879 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1880 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1881 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1882 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1883} 1884 1885//===----------------------------------------------------------------------===// 1886// SVE Floating Point Arithmetic - Predicated Group 1887//===----------------------------------------------------------------------===// 1888 1889class sve_fp_2op_i_p_zds<bits<2> sz, bits<3> opc, string asm, 1890 ZPRRegOp zprty, 1891 Operand imm_ty> 1892: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, imm_ty:$i1), 1893 asm, "\t$Zdn, $Pg/m, $_Zdn, $i1", 1894 "", 1895 []>, Sched<[]> { 1896 bits<3> Pg; 1897 bits<5> Zdn; 1898 bit i1; 1899 let Inst{31-24} = 0b01100101; 1900 let Inst{23-22} = sz; 1901 let Inst{21-19} = 0b011; 1902 let Inst{18-16} = opc; 1903 let Inst{15-13} = 0b100; 1904 let Inst{12-10} = Pg; 1905 let Inst{9-6} = 0b0000; 1906 let Inst{5} = i1; 1907 let Inst{4-0} = Zdn; 1908 1909 let Constraints = "$Zdn = $_Zdn"; 1910 let DestructiveInstType = DestructiveOther; 1911 let ElementSize = zprty.ElementSize; 1912} 1913 1914multiclass sve_fp_2op_i_p_zds<bits<3> opc, string asm, string Ps, Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 1915 let DestructiveInstType = DestructiveBinaryImm in { 1916 def _H : SVEPseudo2Instr<Ps # _H, 1>, sve_fp_2op_i_p_zds<0b01, opc, asm, ZPR16, imm_ty>; 1917 def _S : SVEPseudo2Instr<Ps # _S, 1>, sve_fp_2op_i_p_zds<0b10, opc, asm, ZPR32, imm_ty>; 1918 def _D : SVEPseudo2Instr<Ps # _D, 1>, sve_fp_2op_i_p_zds<0b11, opc, asm, ZPR64, imm_ty>; 1919 } 1920 1921 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_H")>; 1922 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_H")>; 1923 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_S")>; 1924 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_S")>; 1925 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_D")>; 1926 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_D")>; 1927} 1928 1929class sve_fp_2op_p_zds<bits<2> sz, bits<4> opc, string asm, 1930 ZPRRegOp zprty> 1931: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 1932 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 1933 "", 1934 []>, Sched<[]> { 1935 bits<3> Pg; 1936 bits<5> Zdn; 1937 bits<5> Zm; 1938 let Inst{31-24} = 0b01100101; 1939 let Inst{23-22} = sz; 1940 let Inst{21-20} = 0b00; 1941 let Inst{19-16} = opc; 1942 let Inst{15-13} = 0b100; 1943 let Inst{12-10} = Pg; 1944 let Inst{9-5} = Zm; 1945 let Inst{4-0} = Zdn; 1946 1947 let Constraints = "$Zdn = $_Zdn"; 1948 let DestructiveInstType = DestructiveOther; 1949 let ElementSize = zprty.ElementSize; 1950} 1951 1952multiclass sve_fp_2op_p_zds<bits<4> opc, string asm, string Ps, 1953 SDPatternOperator op, DestructiveInstTypeEnum flags, 1954 string revname="", bit isReverseInstr=0> { 1955 let DestructiveInstType = flags in { 1956 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>, 1957 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 1958 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>, 1959 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 1960 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>, 1961 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 1962 } 1963 1964 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 1965 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 1966 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 1967} 1968 1969multiclass sve_fp_2op_p_zds_fscale<bits<4> opc, string asm, 1970 SDPatternOperator op> { 1971 def _H : sve_fp_2op_p_zds<0b01, opc, asm, ZPR16>; 1972 def _S : sve_fp_2op_p_zds<0b10, opc, asm, ZPR32>; 1973 def _D : sve_fp_2op_p_zds<0b11, opc, asm, ZPR64>; 1974 1975 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1976 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1977 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1978} 1979 1980multiclass sve_fp_2op_p_zds_zeroing_hsd<SDPatternOperator op> { 1981 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 1982 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 1983 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 1984 1985 def : SVE_3_Op_Pat_SelZero<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _ZERO_H)>; 1986 def : SVE_3_Op_Pat_SelZero<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _ZERO_S)>; 1987 def : SVE_3_Op_Pat_SelZero<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _ZERO_D)>; 1988} 1989 1990class sve_fp_ftmad<bits<2> sz, string asm, ZPRRegOp zprty> 1991: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, timm32_0_7:$imm3), 1992 asm, "\t$Zdn, $_Zdn, $Zm, $imm3", 1993 "", 1994 []>, Sched<[]> { 1995 bits<5> Zdn; 1996 bits<5> Zm; 1997 bits<3> imm3; 1998 let Inst{31-24} = 0b01100101; 1999 let Inst{23-22} = sz; 2000 let Inst{21-19} = 0b010; 2001 let Inst{18-16} = imm3; 2002 let Inst{15-10} = 0b100000; 2003 let Inst{9-5} = Zm; 2004 let Inst{4-0} = Zdn; 2005 2006 let Constraints = "$Zdn = $_Zdn"; 2007 let DestructiveInstType = DestructiveOther; 2008 let ElementSize = ElementSizeNone; 2009} 2010 2011multiclass sve_fp_ftmad<string asm, SDPatternOperator op> { 2012 def _H : sve_fp_ftmad<0b01, asm, ZPR16>; 2013 def _S : sve_fp_ftmad<0b10, asm, ZPR32>; 2014 def _D : sve_fp_ftmad<0b11, asm, ZPR64>; 2015 2016 def : Pat<(nxv8f16 (op (nxv8f16 ZPR16:$Zn), (nxv8f16 ZPR16:$Zm), (i32 timm32_0_7:$imm))), 2017 (!cast<Instruction>(NAME # _H) ZPR16:$Zn, ZPR16:$Zm, timm32_0_7:$imm)>; 2018 def : Pat<(nxv4f32 (op (nxv4f32 ZPR32:$Zn), (nxv4f32 ZPR32:$Zm), (i32 timm32_0_7:$imm))), 2019 (!cast<Instruction>(NAME # _S) ZPR32:$Zn, ZPR32:$Zm, timm32_0_7:$imm)>; 2020 def : Pat<(nxv2f64 (op (nxv2f64 ZPR64:$Zn), (nxv2f64 ZPR64:$Zm), (i32 timm32_0_7:$imm))), 2021 (!cast<Instruction>(NAME # _D) ZPR64:$Zn, ZPR64:$Zm, timm32_0_7:$imm)>; 2022} 2023 2024multiclass sve_fp_2op_i_p_zds_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator ir_op = null_frag> { 2025 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesUndef>; 2026 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesUndef>; 2027 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesUndef>; 2028 2029 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 2030 def : SVE_2_Op_Fp_Imm_Pat<nxv8f16, ir_op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 2031 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 2032 def : SVE_2_Op_Fp_Imm_Pat<nxv4f16, ir_op, nxv4i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 2033 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, A, 0, !cast<Instruction>(NAME # "_UNDEF_H")>; 2034 def : SVE_2_Op_Fp_Imm_Pat<nxv2f16, ir_op, nxv2i1, f16, B, 1, !cast<Instruction>(NAME # "_UNDEF_H")>; 2035 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 2036 def : SVE_2_Op_Fp_Imm_Pat<nxv4f32, ir_op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 2037 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, A, 0, !cast<Instruction>(NAME # "_UNDEF_S")>; 2038 def : SVE_2_Op_Fp_Imm_Pat<nxv2f32, ir_op, nxv2i1, f32, B, 1, !cast<Instruction>(NAME # "_UNDEF_S")>; 2039 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_UNDEF_D")>; 2040 def : SVE_2_Op_Fp_Imm_Pat<nxv2f64, ir_op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_UNDEF_D")>; 2041} 2042 2043multiclass sve_fp_2op_i_p_zds_zeroing_hfd<Operand imm_ty, FPImmLeaf A, FPImmLeaf B, SDPatternOperator op> { 2044 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, imm_ty, FalseLanesZero>; 2045 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, imm_ty, FalseLanesZero>; 2046 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, imm_ty, FalseLanesZero>; 2047 2048 let AddedComplexity = 2 in { 2049 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, A, 0, !cast<Instruction>(NAME # "_ZERO_H")>; 2050 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv8f16, op, nxv8i1, f16, B, 1, !cast<Instruction>(NAME # "_ZERO_H")>; 2051 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, A, 0, !cast<Instruction>(NAME # "_ZERO_S")>; 2052 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv4f32, op, nxv4i1, f32, B, 1, !cast<Instruction>(NAME # "_ZERO_S")>; 2053 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, A, 0, !cast<Instruction>(NAME # "_ZERO_D")>; 2054 def : SVE_2_Op_Fp_Imm_Pat_Zero<nxv2f64, op, nxv2i1, f64, B, 1, !cast<Instruction>(NAME # "_ZERO_D")>; 2055 } 2056} 2057 2058//===----------------------------------------------------------------------===// 2059// SVE Floating Point Arithmetic - Unpredicated Group 2060//===----------------------------------------------------------------------===// 2061 2062class sve_fp_3op_u_zd<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 2063: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2064 asm, "\t$Zd, $Zn, $Zm", 2065 "", 2066 []>, Sched<[]> { 2067 bits<5> Zd; 2068 bits<5> Zm; 2069 bits<5> Zn; 2070 let Inst{31-24} = 0b01100101; 2071 let Inst{23-22} = sz; 2072 let Inst{21} = 0b0; 2073 let Inst{20-16} = Zm; 2074 let Inst{15-13} = 0b000; 2075 let Inst{12-10} = opc; 2076 let Inst{9-5} = Zn; 2077 let Inst{4-0} = Zd; 2078} 2079 2080multiclass sve_fp_3op_u_zd<bits<3> opc, string asm, SDPatternOperator op, 2081 SDPatternOperator predicated_op = null_frag> { 2082 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2083 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2084 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2085 2086 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2087 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2088 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2089 2090 def : SVE_2_Op_Pred_All_Active<nxv8f16, predicated_op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2091 def : SVE_2_Op_Pred_All_Active<nxv4f32, predicated_op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2092 def : SVE_2_Op_Pred_All_Active<nxv2f64, predicated_op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2093} 2094 2095multiclass sve_fp_3op_u_zd_ftsmul<bits<3> opc, string asm, SDPatternOperator op> { 2096 def _H : sve_fp_3op_u_zd<0b01, opc, asm, ZPR16>; 2097 def _S : sve_fp_3op_u_zd<0b10, opc, asm, ZPR32>; 2098 def _D : sve_fp_3op_u_zd<0b11, opc, asm, ZPR64>; 2099 2100 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2101 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2102 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2103} 2104 2105//===----------------------------------------------------------------------===// 2106// SVE Floating Point Fused Multiply-Add Group 2107//===----------------------------------------------------------------------===// 2108 2109class sve_fp_3op_p_zds_a<bits<2> sz, bits<2> opc, string asm, ZPRRegOp zprty> 2110: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 2111 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 2112 "", 2113 []>, Sched<[]> { 2114 bits<3> Pg; 2115 bits<5> Zda; 2116 bits<5> Zm; 2117 bits<5> Zn; 2118 let Inst{31-24} = 0b01100101; 2119 let Inst{23-22} = sz; 2120 let Inst{21} = 0b1; 2121 let Inst{20-16} = Zm; 2122 let Inst{15} = 0b0; 2123 let Inst{14-13} = opc; 2124 let Inst{12-10} = Pg; 2125 let Inst{9-5} = Zn; 2126 let Inst{4-0} = Zda; 2127 2128 let Constraints = "$Zda = $_Zda"; 2129 let ElementSize = zprty.ElementSize; 2130 let DestructiveInstType = DestructiveTernaryCommWithRev; 2131} 2132 2133multiclass sve_fp_3op_p_zds_a<bits<2> opc, string asm, string Ps, 2134 SDPatternOperator op, string revname, 2135 bit isReverseInstr=0> { 2136 def _H : sve_fp_3op_p_zds_a<0b01, opc, asm, ZPR16>, 2137 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2138 def _S : sve_fp_3op_p_zds_a<0b10, opc, asm, ZPR32>, 2139 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2140 def _D : sve_fp_3op_p_zds_a<0b11, opc, asm, ZPR64>, 2141 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2142 2143 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2144 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2145 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2146} 2147 2148class sve_fp_3op_p_zds_b<bits<2> sz, bits<2> opc, string asm, 2149 ZPRRegOp zprty> 2150: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2151 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2152 "", 2153 []>, Sched<[]> { 2154 bits<3> Pg; 2155 bits<5> Za; 2156 bits<5> Zdn; 2157 bits<5> Zm; 2158 let Inst{31-24} = 0b01100101; 2159 let Inst{23-22} = sz; 2160 let Inst{21} = 0b1; 2161 let Inst{20-16} = Za; 2162 let Inst{15} = 0b1; 2163 let Inst{14-13} = opc; 2164 let Inst{12-10} = Pg; 2165 let Inst{9-5} = Zm; 2166 let Inst{4-0} = Zdn; 2167 2168 let Constraints = "$Zdn = $_Zdn"; 2169 let DestructiveInstType = DestructiveOther; 2170 let ElementSize = zprty.ElementSize; 2171} 2172 2173multiclass sve_fp_3op_p_zds_b<bits<2> opc, string asm, SDPatternOperator op, 2174 string revname, bit isReverseInstr> { 2175 def _H : sve_fp_3op_p_zds_b<0b01, opc, asm, ZPR16>, 2176 SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2177 def _S : sve_fp_3op_p_zds_b<0b10, opc, asm, ZPR32>, 2178 SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2179 def _D : sve_fp_3op_p_zds_b<0b11, opc, asm, ZPR64>, 2180 SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2181 2182 def : SVE_4_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2183 def : SVE_4_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2184 def : SVE_4_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2185} 2186 2187multiclass sve_fp_3op_p_zds_zx { 2188 def _UNDEF_H : PredThreeOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 2189 def _UNDEF_S : PredThreeOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 2190 def _UNDEF_D : PredThreeOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 2191} 2192 2193//===----------------------------------------------------------------------===// 2194// SVE Floating Point Multiply-Add - Indexed Group 2195//===----------------------------------------------------------------------===// 2196 2197class sve_fp_fma_by_indexed_elem<bits<2> sz, bits<2> opc, string asm, 2198 ZPRRegOp zprty1, 2199 ZPRRegOp zprty2, Operand itype> 2200: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty1:$Zn, zprty2:$Zm, itype:$iop), 2201 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 2202 bits<5> Zda; 2203 bits<5> Zn; 2204 let Inst{31-24} = 0b01100100; 2205 let Inst{23-22} = sz; 2206 let Inst{21} = 0b1; 2207 let Inst{15-12} = 0b0000; 2208 let Inst{11-10} = opc; 2209 let Inst{9-5} = Zn; 2210 let Inst{4-0} = Zda; 2211 2212 let Constraints = "$Zda = $_Zda"; 2213 let DestructiveInstType = DestructiveOther; 2214 let ElementSize = ElementSizeNone; 2215} 2216 2217multiclass sve2p1_fp_bfma_by_indexed_elem<string asm, bits<2> opc> { 2218 def NAME : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, 2219 VectorIndexH32b> { 2220 bits<3> Zm; 2221 bits<3> iop; 2222 let Inst{22} = iop{2}; 2223 let Inst{20-19} = iop{1-0}; 2224 let Inst{18-16} = Zm; 2225 } 2226} 2227 2228multiclass sve_fp_fma_by_indexed_elem<bits<2> opc, string asm, 2229 SDPatternOperator op> { 2230 def _H : sve_fp_fma_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2231 bits<3> Zm; 2232 bits<3> iop; 2233 let Inst{22} = iop{2}; 2234 let Inst{20-19} = iop{1-0}; 2235 let Inst{18-16} = Zm; 2236 } 2237 def _S : sve_fp_fma_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2238 bits<3> Zm; 2239 bits<2> iop; 2240 let Inst{20-19} = iop; 2241 let Inst{18-16} = Zm; 2242 } 2243 def _D : sve_fp_fma_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2244 bits<4> Zm; 2245 bit iop; 2246 let Inst{20} = iop; 2247 let Inst{19-16} = Zm; 2248 } 2249 2250 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexH32b_timm:$idx))), 2251 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexH32b_timm:$idx)>; 2252 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexS32b_timm:$idx))), 2253 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx)>; 2254 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 VectorIndexD32b_timm:$idx))), 2255 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx)>; 2256} 2257 2258 2259//===----------------------------------------------------------------------===// 2260// SVE Floating Point Multiply - Indexed Group 2261//===----------------------------------------------------------------------===// 2262 2263class sve_fp_fmul_by_indexed_elem<bits<2> sz, bit o2, string asm, ZPRRegOp zprty, 2264 ZPRRegOp zprty2, Operand itype> 2265: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty2:$Zm, itype:$iop), 2266 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 2267 bits<5> Zd; 2268 bits<5> Zn; 2269 let Inst{31-24} = 0b01100100; 2270 let Inst{23-22} = sz; 2271 let Inst{21} = 0b1; 2272 let Inst{15-12} = 0b0010; 2273 let Inst{11} = o2; 2274 let Inst{10} = 0b0; 2275 let Inst{9-5} = Zn; 2276 let Inst{4-0} = Zd; 2277} 2278 2279multiclass sve2p1_fp_bfmul_by_indexed_elem<string asm> { 2280 def NAME : sve_fp_fmul_by_indexed_elem<{0, ?}, 0b1, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2281 bits<3> Zm; 2282 bits<3> iop; 2283 let Inst{22} = iop{2}; 2284 let Inst{20-19} = iop{1-0}; 2285 let Inst{18-16} = Zm; 2286 } 2287} 2288 2289multiclass sve_fp_fmul_by_indexed_elem<string asm, SDPatternOperator op> { 2290 def _H : sve_fp_fmul_by_indexed_elem<{0, ?}, 0b0, asm, ZPR16, ZPR3b16, VectorIndexH32b> { 2291 bits<3> Zm; 2292 bits<3> iop; 2293 let Inst{22} = iop{2}; 2294 let Inst{20-19} = iop{1-0}; 2295 let Inst{18-16} = Zm; 2296 } 2297 def _S : sve_fp_fmul_by_indexed_elem<0b10, 0b0, asm, ZPR32, ZPR3b32, VectorIndexS32b> { 2298 bits<3> Zm; 2299 bits<2> iop; 2300 let Inst{20-19} = iop; 2301 let Inst{18-16} = Zm; 2302 } 2303 def _D : sve_fp_fmul_by_indexed_elem<0b11, 0b0, asm, ZPR64, ZPR4b64, VectorIndexD32b> { 2304 bits<4> Zm; 2305 bit iop; 2306 let Inst{20} = iop; 2307 let Inst{19-16} = Zm; 2308 } 2309 2310 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, (i32 VectorIndexH32b_timm:$idx))), 2311 (!cast<Instruction>(NAME # _H) $Op1, $Op2, VectorIndexH32b_timm:$idx)>; 2312 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, (i32 VectorIndexS32b_timm:$idx))), 2313 (!cast<Instruction>(NAME # _S) $Op1, $Op2, VectorIndexS32b_timm:$idx)>; 2314 def : Pat<(nxv2f64 (op nxv2f64:$Op1, nxv2f64:$Op2, (i32 VectorIndexD32b_timm:$idx))), 2315 (!cast<Instruction>(NAME # _D) $Op1, $Op2, VectorIndexD32b_timm:$idx)>; 2316} 2317 2318//===----------------------------------------------------------------------===// 2319// SVE Floating Point Complex Multiply-Add Group 2320//===----------------------------------------------------------------------===// 2321 2322class sve_fp_fcmla<bits<2> sz, string asm, ZPRRegOp zprty> 2323: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm, 2324 complexrotateop:$imm), 2325 asm, "\t$Zda, $Pg/m, $Zn, $Zm, $imm", 2326 "", []>, Sched<[]> { 2327 bits<5> Zda; 2328 bits<3> Pg; 2329 bits<5> Zn; 2330 bits<5> Zm; 2331 bits<2> imm; 2332 let Inst{31-24} = 0b01100100; 2333 let Inst{23-22} = sz; 2334 let Inst{21} = 0; 2335 let Inst{20-16} = Zm; 2336 let Inst{15} = 0; 2337 let Inst{14-13} = imm; 2338 let Inst{12-10} = Pg; 2339 let Inst{9-5} = Zn; 2340 let Inst{4-0} = Zda; 2341 2342 let Constraints = "$Zda = $_Zda"; 2343 let DestructiveInstType = DestructiveOther; 2344 let ElementSize = zprty.ElementSize; 2345} 2346 2347multiclass sve_fp_fcmla<string asm, SDPatternOperator op> { 2348 def _H : sve_fp_fcmla<0b01, asm, ZPR16>; 2349 def _S : sve_fp_fcmla<0b10, asm, ZPR32>; 2350 def _D : sve_fp_fcmla<0b11, asm, ZPR64>; 2351 2352 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, nxv8f16:$Op4, (i32 complexrotateop:$imm))), 2353 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2354 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, nxv4f32:$Op4, (i32 complexrotateop:$imm))), 2355 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2356 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, nxv2f64:$Op4, (i32 complexrotateop:$imm))), 2357 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, $Op4, complexrotateop:$imm)>; 2358} 2359 2360//===----------------------------------------------------------------------===// 2361// SVE Floating Point Complex Multiply-Add - Indexed Group 2362//===----------------------------------------------------------------------===// 2363 2364class sve_fp_fcmla_by_indexed_elem<bits<2> sz, string asm, 2365 ZPRRegOp zprty, 2366 ZPRRegOp zprty2, Operand itype> 2367: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty2:$Zm, itype:$iop, 2368 complexrotateop:$imm), 2369 asm, "\t$Zda, $Zn, $Zm$iop, $imm", 2370 "", []>, Sched<[]> { 2371 bits<5> Zda; 2372 bits<5> Zn; 2373 bits<2> imm; 2374 let Inst{31-24} = 0b01100100; 2375 let Inst{23-22} = sz; 2376 let Inst{21} = 0b1; 2377 let Inst{15-12} = 0b0001; 2378 let Inst{11-10} = imm; 2379 let Inst{9-5} = Zn; 2380 let Inst{4-0} = Zda; 2381 2382 let Constraints = "$Zda = $_Zda"; 2383 let DestructiveInstType = DestructiveOther; 2384 let ElementSize = ElementSizeNone; 2385} 2386 2387multiclass sve_fp_fcmla_by_indexed_elem<string asm, SDPatternOperator op> { 2388 def _H : sve_fp_fcmla_by_indexed_elem<0b10, asm, ZPR16, ZPR3b16, VectorIndexS32b> { 2389 bits<3> Zm; 2390 bits<2> iop; 2391 let Inst{20-19} = iop; 2392 let Inst{18-16} = Zm; 2393 } 2394 def _S : sve_fp_fcmla_by_indexed_elem<0b11, asm, ZPR32, ZPR4b32, VectorIndexD32b> { 2395 bits<4> Zm; 2396 bits<1> iop; 2397 let Inst{20} = iop; 2398 let Inst{19-16} = Zm; 2399 } 2400 2401 def : Pat<(nxv8f16 (op nxv8f16:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 2402 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 2403 def : Pat<(nxv4f32 (op nxv4f32:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 2404 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 2405} 2406 2407//===----------------------------------------------------------------------===// 2408// SVE Floating Point Complex Addition Group 2409//===----------------------------------------------------------------------===// 2410 2411class sve_fp_fcadd<bits<2> sz, string asm, ZPRRegOp zprty> 2412: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, 2413 complexrotateopodd:$imm), 2414 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm, $imm", 2415 "", 2416 []>, Sched<[]> { 2417 bits<5> Zdn; 2418 bits<5> Zm; 2419 bits<3> Pg; 2420 bit imm; 2421 let Inst{31-24} = 0b01100100; 2422 let Inst{23-22} = sz; 2423 let Inst{21-17} = 0; 2424 let Inst{16} = imm; 2425 let Inst{15-13} = 0b100; 2426 let Inst{12-10} = Pg; 2427 let Inst{9-5} = Zm; 2428 let Inst{4-0} = Zdn; 2429 2430 let Constraints = "$Zdn = $_Zdn"; 2431 let DestructiveInstType = DestructiveOther; 2432 let ElementSize = zprty.ElementSize; 2433} 2434 2435multiclass sve_fp_fcadd<string asm, SDPatternOperator op> { 2436 def _H : sve_fp_fcadd<0b01, asm, ZPR16>; 2437 def _S : sve_fp_fcadd<0b10, asm, ZPR32>; 2438 def _D : sve_fp_fcadd<0b11, asm, ZPR64>; 2439 2440 def : Pat<(nxv8f16 (op nxv8i1:$Op1, nxv8f16:$Op2, nxv8f16:$Op3, (i32 complexrotateopodd:$imm))), 2441 (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2442 def : Pat<(nxv4f32 (op nxv4i1:$Op1, nxv4f32:$Op2, nxv4f32:$Op3, (i32 complexrotateopodd:$imm))), 2443 (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2444 def : Pat<(nxv2f64 (op nxv2i1:$Op1, nxv2f64:$Op2, nxv2f64:$Op3, (i32 complexrotateopodd:$imm))), 2445 (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3, complexrotateopodd:$imm)>; 2446} 2447 2448//===----------------------------------------------------------------------===// 2449// SVE2 Floating Point Convert Group 2450//===----------------------------------------------------------------------===// 2451 2452class sve2_fp_convert_precision<bits<4> opc, string asm, 2453 ZPRRegOp zprty1, ZPRRegOp zprty2> 2454: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, PPR3bAny:$Pg, zprty2:$Zn), 2455 asm, "\t$Zd, $Pg/m, $Zn", 2456 "", 2457 []>, Sched<[]> { 2458 bits<5> Zd; 2459 bits<5> Zn; 2460 bits<3> Pg; 2461 let Inst{31-24} = 0b01100100; 2462 let Inst{23-22} = opc{3-2}; 2463 let Inst{21-18} = 0b0010; 2464 let Inst{17-16} = opc{1-0}; 2465 let Inst{15-13} = 0b101; 2466 let Inst{12-10} = Pg; 2467 let Inst{9-5} = Zn; 2468 let Inst{4-0} = Zd; 2469 2470 let Constraints = "$Zd = $_Zd"; 2471} 2472 2473multiclass sve2_fp_convert_down_narrow<string asm, string op> { 2474 def _StoH : sve2_fp_convert_precision<0b1000, asm, ZPR16, ZPR32>; 2475 def _DtoS : sve2_fp_convert_precision<0b1110, asm, ZPR32, ZPR64>; 2476 2477 def : SVE_3_Op_Pat<nxv8f16, !cast<SDPatternOperator>(op # _f16f32), nxv8f16, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _StoH)>; 2478 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2479} 2480 2481multiclass sve2_fp_convert_up_long<string asm, string op> { 2482 def _HtoS : sve2_fp_convert_precision<0b1001, asm, ZPR32, ZPR16>; 2483 def _StoD : sve2_fp_convert_precision<0b1111, asm, ZPR64, ZPR32>; 2484 2485 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f16), nxv4f32, nxv4i1, nxv8f16, !cast<Instruction>(NAME # _HtoS)>; 2486 def : SVE_3_Op_Pat<nxv2f64, !cast<SDPatternOperator>(op # _f64f32), nxv2f64, nxv2i1, nxv4f32, !cast<Instruction>(NAME # _StoD)>; 2487} 2488 2489multiclass sve2_fp_convert_down_odd_rounding_top<string asm, string op> { 2490 def _DtoS : sve2_fp_convert_precision<0b0010, asm, ZPR32, ZPR64>; 2491 2492 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2493} 2494 2495//===----------------------------------------------------------------------===// 2496// SVE2 Floating Point Pairwise Group 2497//===----------------------------------------------------------------------===// 2498 2499class sve2_fp_pairwise_pred<bits<2> sz, bits<3> opc, string asm, 2500 ZPRRegOp zprty> 2501: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2502 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 2503 "", 2504 []>, Sched<[]> { 2505 bits<3> Pg; 2506 bits<5> Zm; 2507 bits<5> Zdn; 2508 let Inst{31-24} = 0b01100100; 2509 let Inst{23-22} = sz; 2510 let Inst{21-19} = 0b010; 2511 let Inst{18-16} = opc; 2512 let Inst{15-13} = 0b100; 2513 let Inst{12-10} = Pg; 2514 let Inst{9-5} = Zm; 2515 let Inst{4-0} = Zdn; 2516 2517 let Constraints = "$Zdn = $_Zdn"; 2518 let DestructiveInstType = DestructiveOther; 2519 let ElementSize = zprty.ElementSize; 2520} 2521 2522multiclass sve2_fp_pairwise_pred<bits<3> opc, string asm, 2523 SDPatternOperator op> { 2524 def _H : sve2_fp_pairwise_pred<0b01, opc, asm, ZPR16>; 2525 def _S : sve2_fp_pairwise_pred<0b10, opc, asm, ZPR32>; 2526 def _D : sve2_fp_pairwise_pred<0b11, opc, asm, ZPR64>; 2527 2528 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2529 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2530 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2531} 2532 2533//===----------------------------------------------------------------------===// 2534// SVE2 Floating Point Widening Multiply-Add - Indexed Group 2535//===----------------------------------------------------------------------===// 2536 2537class sve2_fp_mla_long_by_indexed_elem<bits<3> opc, string asm> 2538: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, 2539 VectorIndexH32b:$iop), 2540 asm, "\t$Zda, $Zn, $Zm$iop", 2541 "", 2542 []>, Sched<[]> { 2543 bits<5> Zda; 2544 bits<5> Zn; 2545 bits<3> Zm; 2546 bits<3> iop; 2547 let Inst{31-23} = 0b011001001; 2548 let Inst{22} = opc{2}; 2549 let Inst{21} = 0b1; 2550 let Inst{20-19} = iop{2-1}; 2551 let Inst{18-16} = Zm; 2552 let Inst{15-14} = 0b01; 2553 let Inst{13} = opc{1}; 2554 let Inst{12} = 0b0; 2555 let Inst{11} = iop{0}; 2556 let Inst{10} = opc{0}; 2557 let Inst{9-5} = Zn; 2558 let Inst{4-0} = Zda; 2559 2560 let Constraints = "$Zda = $_Zda"; 2561 let DestructiveInstType = DestructiveOther; 2562 let ElementSize = ElementSizeNone; 2563} 2564 2565multiclass sve2_fp_mla_long_by_indexed_elem<bits<3> opc, string asm, 2566 ValueType OutVT, ValueType InVT, 2567 SDPatternOperator op> { 2568 def NAME : sve2_fp_mla_long_by_indexed_elem<opc, asm>; 2569 def : SVE_4_Op_Imm_Pat<OutVT, op, OutVT, InVT, InVT, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME)>; 2570} 2571 2572//===----------------------------------------------------------------------===// 2573// SVE2 Floating Point Widening Multiply-Add Group 2574//===----------------------------------------------------------------------===// 2575 2576class sve2_fp_mla_long<bits<3> opc, string asm> 2577: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 2578 asm, "\t$Zda, $Zn, $Zm", 2579 "", 2580 []>, Sched<[]> { 2581 bits<5> Zda; 2582 bits<5> Zn; 2583 bits<5> Zm; 2584 let Inst{31-23} = 0b011001001; 2585 let Inst{22} = opc{2}; 2586 let Inst{21} = 0b1; 2587 let Inst{20-16} = Zm; 2588 let Inst{15-14} = 0b10; 2589 let Inst{13} = opc{1}; 2590 let Inst{12-11} = 0b00; 2591 let Inst{10} = opc{0}; 2592 let Inst{9-5} = Zn; 2593 let Inst{4-0} = Zda; 2594 2595 let Constraints = "$Zda = $_Zda"; 2596 let DestructiveInstType = DestructiveOther; 2597 let ElementSize = ElementSizeNone; 2598} 2599 2600multiclass sve2_fp_mla_long<bits<3> opc, string asm, ValueType OutVT, 2601 ValueType InVT, SDPatternOperator op> { 2602 def NAME : sve2_fp_mla_long<opc, asm>; 2603 def : SVE_3_Op_Pat<OutVT, op, OutVT, InVT, InVT, !cast<Instruction>(NAME)>; 2604} 2605 2606//===----------------------------------------------------------------------===// 2607// SVE Stack Allocation Group 2608//===----------------------------------------------------------------------===// 2609 2610class sve_int_arith_vl<bit opc, string asm, bit streaming_sve = 0b0> 2611: I<(outs GPR64sp:$Rd), (ins GPR64sp:$Rn, simm6_32b:$imm6), 2612 asm, "\t$Rd, $Rn, $imm6", 2613 "", 2614 []>, Sched<[]> { 2615 bits<5> Rd; 2616 bits<5> Rn; 2617 bits<6> imm6; 2618 let Inst{31-23} = 0b000001000; 2619 let Inst{22} = opc; 2620 let Inst{21} = 0b1; 2621 let Inst{20-16} = Rn; 2622 let Inst{15-12} = 0b0101; 2623 let Inst{11} = streaming_sve; 2624 let Inst{10-5} = imm6; 2625 let Inst{4-0} = Rd; 2626} 2627 2628class sve_int_read_vl_a<bit op, bits<5> opc2, string asm, bit streaming_sve = 0b0> 2629: I<(outs GPR64:$Rd), (ins simm6_32b:$imm6), 2630 asm, "\t$Rd, $imm6", 2631 "", 2632 []>, Sched<[]> { 2633 bits<5> Rd; 2634 bits<6> imm6; 2635 let Inst{31-23} = 0b000001001; 2636 let Inst{22} = op; 2637 let Inst{21} = 0b1; 2638 let Inst{20-16} = opc2{4-0}; 2639 let Inst{15-12} = 0b0101; 2640 let Inst{11} = streaming_sve; 2641 let Inst{10-5} = imm6; 2642 let Inst{4-0} = Rd; 2643 2644 let isReMaterializable = 1; 2645} 2646 2647//===----------------------------------------------------------------------===// 2648// SVE Permute - In Lane Group 2649//===----------------------------------------------------------------------===// 2650 2651class sve_int_perm_bin_perm_zz<bits<3> opc, bits<2> sz8_64, string asm, 2652 ZPRRegOp zprty> 2653: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 2654 asm, "\t$Zd, $Zn, $Zm", 2655 "", 2656 []>, Sched<[]> { 2657 bits<5> Zd; 2658 bits<5> Zm; 2659 bits<5> Zn; 2660 let Inst{31-24} = 0b00000101; 2661 let Inst{23-22} = sz8_64; 2662 let Inst{21} = 0b1; 2663 let Inst{20-16} = Zm; 2664 let Inst{15-13} = 0b011; 2665 let Inst{12-10} = opc; 2666 let Inst{9-5} = Zn; 2667 let Inst{4-0} = Zd; 2668} 2669 2670multiclass sve_int_perm_bin_perm_zz<bits<3> opc, string asm, 2671 SDPatternOperator op> { 2672 def _B : sve_int_perm_bin_perm_zz<opc, 0b00, asm, ZPR8>; 2673 def _H : sve_int_perm_bin_perm_zz<opc, 0b01, asm, ZPR16>; 2674 def _S : sve_int_perm_bin_perm_zz<opc, 0b10, asm, ZPR32>; 2675 def _D : sve_int_perm_bin_perm_zz<opc, 0b11, asm, ZPR64>; 2676 2677 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2678 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2679 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2680 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2681 2682 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 2683 def : SVE_2_Op_Pat<nxv4f16, op, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _S)>; 2684 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 2685 def : SVE_2_Op_Pat<nxv2f16, op, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _D)>; 2686 def : SVE_2_Op_Pat<nxv2f32, op, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _D)>; 2687 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 2688 2689 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 2690} 2691 2692//===----------------------------------------------------------------------===// 2693// SVE Floating Point Unary Operations Group 2694//===----------------------------------------------------------------------===// 2695 2696class sve_fp_2op_p_zd<bits<7> opc, string asm, RegisterOperand i_zprtype, 2697 RegisterOperand o_zprtype, ElementSizeEnum Sz> 2698: I<(outs o_zprtype:$Zd), (ins i_zprtype:$_Zd, PPR3bAny:$Pg, i_zprtype:$Zn), 2699 asm, "\t$Zd, $Pg/m, $Zn", 2700 "", 2701 []>, Sched<[]> { 2702 bits<3> Pg; 2703 bits<5> Zd; 2704 bits<5> Zn; 2705 let Inst{31-24} = 0b01100101; 2706 let Inst{23-22} = opc{6-5}; 2707 let Inst{21} = 0b0; 2708 let Inst{20-16} = opc{4-0}; 2709 let Inst{15-13} = 0b101; 2710 let Inst{12-10} = Pg; 2711 let Inst{9-5} = Zn; 2712 let Inst{4-0} = Zd; 2713 2714 let Constraints = "$Zd = $_Zd"; 2715 let DestructiveInstType = DestructiveUnaryPassthru; 2716 let ElementSize = Sz; 2717} 2718 2719multiclass sve_fp_2op_p_zd<bits<7> opc, string asm, 2720 RegisterOperand i_zprtype, 2721 RegisterOperand o_zprtype, 2722 SDPatternOperator int_op, 2723 SDPatternOperator ir_op, ValueType vt1, 2724 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2725 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2726 SVEPseudo2Instr<NAME, 1>; 2727 // convert vt1 to a packed type for the intrinsic patterns 2728 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2729 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2730 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2731 1 : vt1); 2732 2733 // convert vt3 to a packed type for the intrinsic patterns 2734 defvar packedvt3 = !cond(!eq(!cast<string>(vt3), "nxv2f16"): nxv8f16, 2735 !eq(!cast<string>(vt3), "nxv4f16"): nxv8f16, 2736 !eq(!cast<string>(vt3), "nxv2f32"): nxv4f32, 2737 1 : vt3); 2738 2739 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, packedvt3, !cast<Instruction>(NAME)>; 2740 def : SVE_1_Op_Passthru_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2741 2742 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2743 2744 defm : SVE_1_Op_PassthruUndef_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2745} 2746 2747multiclass sve_fp_2op_p_zdr<bits<7> opc, string asm, 2748 RegisterOperand i_zprtype, 2749 RegisterOperand o_zprtype, 2750 SDPatternOperator int_op, 2751 SDPatternOperator ir_op, ValueType vt1, 2752 ValueType vt2, ValueType vt3, ElementSizeEnum Sz> { 2753 def NAME : sve_fp_2op_p_zd<opc, asm, i_zprtype, o_zprtype, Sz>, 2754 SVEPseudo2Instr<NAME, 1>; 2755 2756 // convert vt1 to a packed type for the intrinsic patterns 2757 defvar packedvt1 = !cond(!eq(!cast<string>(vt1), "nxv2f16"): nxv8f16, 2758 !eq(!cast<string>(vt1), "nxv4f16"): nxv8f16, 2759 !eq(!cast<string>(vt1), "nxv2f32"): nxv4f32, 2760 1 : vt1); 2761 2762 def : SVE_3_Op_Pat<packedvt1, int_op, packedvt1, vt2, vt3, !cast<Instruction>(NAME)>; 2763 def : SVE_1_Op_Passthru_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME)>; 2764 2765 def _UNDEF : PredOneOpPassthruPseudo<NAME, !cast<ZPRRegOp>(i_zprtype)>; 2766 2767 defm : SVE_1_Op_PassthruUndef_Round_Pat<vt1, ir_op, vt2, vt3, !cast<Instruction>(NAME # _UNDEF)>; 2768} 2769 2770multiclass sve_fp_2op_p_zd_HSD<bits<5> opc, string asm, SDPatternOperator op> { 2771 def _H : sve_fp_2op_p_zd<{ 0b01, opc }, asm, ZPR16, ZPR16, ElementSizeH>, 2772 SVEPseudo2Instr<NAME # _H, 1>; 2773 def _S : sve_fp_2op_p_zd<{ 0b10, opc }, asm, ZPR32, ZPR32, ElementSizeS>, 2774 SVEPseudo2Instr<NAME # _S, 1>; 2775 def _D : sve_fp_2op_p_zd<{ 0b11, opc }, asm, ZPR64, ZPR64, ElementSizeD>, 2776 SVEPseudo2Instr<NAME # _D, 1>; 2777 2778 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2779 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 2780 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 2781 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2782 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 2783 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2784 2785 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 2786 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 2787 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 2788 2789 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2790 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2791 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _UNDEF_H)>; 2792 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2793 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _UNDEF_S)>; 2794 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _UNDEF_D)>; 2795} 2796 2797multiclass sve2_fp_flogb<string asm, SDPatternOperator op> { 2798 def _H : sve_fp_2op_p_zd<0b0011010, asm, ZPR16, ZPR16, ElementSizeH>; 2799 def _S : sve_fp_2op_p_zd<0b0011100, asm, ZPR32, ZPR32, ElementSizeS>; 2800 def _D : sve_fp_2op_p_zd<0b0011110, asm, ZPR64, ZPR64, ElementSizeD>; 2801 2802 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 2803 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 2804 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 2805} 2806 2807multiclass sve2_fp_convert_down_odd_rounding<string asm, string op> { 2808 def _DtoS : sve_fp_2op_p_zd<0b0001010, asm, ZPR64, ZPR32, ElementSizeD>; 2809 def : SVE_3_Op_Pat<nxv4f32, !cast<SDPatternOperator>(op # _f32f64), nxv4f32, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _DtoS)>; 2810} 2811 2812//===----------------------------------------------------------------------===// 2813// SVE Floating Point Unary Operations - Unpredicated Group 2814//===----------------------------------------------------------------------===// 2815 2816class sve_fp_2op_u_zd<bits<2> sz, bits<3> opc, string asm, 2817 ZPRRegOp zprty> 2818: I<(outs zprty:$Zd), (ins zprty:$Zn), 2819 asm, "\t$Zd, $Zn", 2820 "", 2821 []>, Sched<[]> { 2822 bits<5> Zd; 2823 bits<5> Zn; 2824 let Inst{31-24} = 0b01100101; 2825 let Inst{23-22} = sz; 2826 let Inst{21-19} = 0b001; 2827 let Inst{18-16} = opc; 2828 let Inst{15-10} = 0b001100; 2829 let Inst{9-5} = Zn; 2830 let Inst{4-0} = Zd; 2831} 2832 2833multiclass sve_fp_2op_u_zd<bits<3> opc, string asm, SDPatternOperator op> { 2834 def _H : sve_fp_2op_u_zd<0b01, opc, asm, ZPR16>; 2835 def _S : sve_fp_2op_u_zd<0b10, opc, asm, ZPR32>; 2836 def _D : sve_fp_2op_u_zd<0b11, opc, asm, ZPR64>; 2837 2838 def : SVE_1_Op_Pat<nxv8f16, op, nxv8f16, !cast<Instruction>(NAME # _H)>; 2839 def : SVE_1_Op_Pat<nxv4f32, op, nxv4f32, !cast<Instruction>(NAME # _S)>; 2840 def : SVE_1_Op_Pat<nxv2f64, op, nxv2f64, !cast<Instruction>(NAME # _D)>; 2841} 2842 2843//===----------------------------------------------------------------------===// 2844// SVE Integer Arithmetic - Binary Predicated Group 2845//===----------------------------------------------------------------------===// 2846 2847class sve_int_bin_pred_arit_log<bits<2> sz8_64, bits<2> fmt, bits<3> opc, 2848 string asm, ZPRRegOp zprty> 2849: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 2850 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 2851 bits<3> Pg; 2852 bits<5> Zdn; 2853 bits<5> Zm; 2854 let Inst{31-24} = 0b00000100; 2855 let Inst{23-22} = sz8_64; 2856 let Inst{21} = 0b0; 2857 let Inst{20-19} = fmt; 2858 let Inst{18-16} = opc; 2859 let Inst{15-13} = 0b000; 2860 let Inst{12-10} = Pg; 2861 let Inst{9-5} = Zm; 2862 let Inst{4-0} = Zdn; 2863 2864 let Constraints = "$Zdn = $_Zdn"; 2865 let DestructiveInstType = DestructiveOther; 2866 let ElementSize = zprty.ElementSize; 2867} 2868 2869multiclass sve_int_bin_pred_log<bits<3> opc, string asm, string Ps, 2870 SDPatternOperator op, 2871 DestructiveInstTypeEnum flags> { 2872 let DestructiveInstType = flags in { 2873 def _B : sve_int_bin_pred_arit_log<0b00, 0b11, opc, asm, ZPR8>, 2874 SVEPseudo2Instr<Ps # _B, 1>; 2875 def _H : sve_int_bin_pred_arit_log<0b01, 0b11, opc, asm, ZPR16>, 2876 SVEPseudo2Instr<Ps # _H, 1>; 2877 def _S : sve_int_bin_pred_arit_log<0b10, 0b11, opc, asm, ZPR32>, 2878 SVEPseudo2Instr<Ps # _S, 1>; 2879 def _D : sve_int_bin_pred_arit_log<0b11, 0b11, opc, asm, ZPR64>, 2880 SVEPseudo2Instr<Ps # _D, 1>; 2881 } 2882 2883 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2884 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2885 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2886 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2887} 2888 2889multiclass sve_int_bin_pred_arit_0<bits<3> opc, string asm, string Ps, 2890 SDPatternOperator op, 2891 DestructiveInstTypeEnum flags, 2892 string revname="", bit isReverseInstr=0> { 2893 let DestructiveInstType = flags in { 2894 def _B : sve_int_bin_pred_arit_log<0b00, 0b00, opc, asm, ZPR8>, 2895 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 2896 def _H : sve_int_bin_pred_arit_log<0b01, 0b00, opc, asm, ZPR16>, 2897 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 2898 def _S : sve_int_bin_pred_arit_log<0b10, 0b00, opc, asm, ZPR32>, 2899 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2900 def _D : sve_int_bin_pred_arit_log<0b11, 0b00, opc, asm, ZPR64>, 2901 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2902 } 2903 2904 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2905 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2906 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2907 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2908} 2909 2910multiclass sve_int_bin_pred_arit_1<bits<3> opc, string asm, string Ps, 2911 SDPatternOperator op, 2912 DestructiveInstTypeEnum flags> { 2913 let DestructiveInstType = flags in { 2914 def _B : sve_int_bin_pred_arit_log<0b00, 0b01, opc, asm, ZPR8>, 2915 SVEPseudo2Instr<Ps # _B, 1>; 2916 def _H : sve_int_bin_pred_arit_log<0b01, 0b01, opc, asm, ZPR16>, 2917 SVEPseudo2Instr<Ps # _H, 1>; 2918 def _S : sve_int_bin_pred_arit_log<0b10, 0b01, opc, asm, ZPR32>, 2919 SVEPseudo2Instr<Ps # _S, 1>; 2920 def _D : sve_int_bin_pred_arit_log<0b11, 0b01, opc, asm, ZPR64>, 2921 SVEPseudo2Instr<Ps # _D, 1>; 2922 } 2923 2924 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2925 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2926 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2927 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2928} 2929 2930multiclass sve_int_bin_pred_arit_2<bits<3> opc, string asm, string Ps, 2931 SDPatternOperator op, 2932 DestructiveInstTypeEnum flags> { 2933 let DestructiveInstType = flags in { 2934 def _B : sve_int_bin_pred_arit_log<0b00, 0b10, opc, asm, ZPR8>, 2935 SVEPseudo2Instr<Ps # _B, 1>; 2936 def _H : sve_int_bin_pred_arit_log<0b01, 0b10, opc, asm, ZPR16>, 2937 SVEPseudo2Instr<Ps # _H, 1>; 2938 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2939 SVEPseudo2Instr<Ps # _S, 1>; 2940 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2941 SVEPseudo2Instr<Ps # _D, 1>; 2942 } 2943 2944 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 2945 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 2946 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2947 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2948} 2949 2950// Special case for divides which are not defined for 8b/16b elements. 2951multiclass sve_int_bin_pred_arit_2_div<bits<3> opc, string asm, string Ps, 2952 SDPatternOperator op, 2953 DestructiveInstTypeEnum flags, 2954 string revname="", bit isReverseInstr=0> { 2955 let DestructiveInstType = flags in { 2956 def _S : sve_int_bin_pred_arit_log<0b10, 0b10, opc, asm, ZPR32>, 2957 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 2958 def _D : sve_int_bin_pred_arit_log<0b11, 0b10, opc, asm, ZPR64>, 2959 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 2960 } 2961 2962 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 2963 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 2964} 2965 2966//===----------------------------------------------------------------------===// 2967// SVE Integer Multiply-Add Group 2968//===----------------------------------------------------------------------===// 2969 2970class sve_int_mladdsub_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 2971 ZPRRegOp zprty> 2972: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm, zprty:$Za), 2973 asm, "\t$Zdn, $Pg/m, $Zm, $Za", 2974 "", 2975 []>, Sched<[]> { 2976 bits<3> Pg; 2977 bits<5> Zdn; 2978 bits<5> Za; 2979 bits<5> Zm; 2980 let Inst{31-24} = 0b00000100; 2981 let Inst{23-22} = sz8_64; 2982 let Inst{21} = 0b0; 2983 let Inst{20-16} = Zm; 2984 let Inst{15-14} = 0b11; 2985 let Inst{13} = opc; 2986 let Inst{12-10} = Pg; 2987 let Inst{9-5} = Za; 2988 let Inst{4-0} = Zdn; 2989 2990 let Constraints = "$Zdn = $_Zdn"; 2991 let DestructiveInstType = DestructiveOther; 2992 let ElementSize = zprty.ElementSize; 2993} 2994 2995multiclass sve_int_mladdsub_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 2996 def _B : sve_int_mladdsub_vvv_pred<0b00, opc, asm, ZPR8>; 2997 def _H : sve_int_mladdsub_vvv_pred<0b01, opc, asm, ZPR16>; 2998 def _S : sve_int_mladdsub_vvv_pred<0b10, opc, asm, ZPR32>; 2999 def _D : sve_int_mladdsub_vvv_pred<0b11, opc, asm, ZPR64>; 3000 3001 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3002 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3003 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3004 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3005} 3006 3007class sve_int_mlas_vvv_pred<bits<2> sz8_64, bits<1> opc, string asm, 3008 ZPRRegOp zprty> 3009: I<(outs zprty:$Zda), (ins PPR3bAny:$Pg, zprty:$_Zda, zprty:$Zn, zprty:$Zm), 3010 asm, "\t$Zda, $Pg/m, $Zn, $Zm", 3011 "", 3012 []>, Sched<[]> { 3013 bits<3> Pg; 3014 bits<5> Zda; 3015 bits<5> Zm; 3016 bits<5> Zn; 3017 let Inst{31-24} = 0b00000100; 3018 let Inst{23-22} = sz8_64; 3019 let Inst{21} = 0b0; 3020 let Inst{20-16} = Zm; 3021 let Inst{15-14} = 0b01; 3022 let Inst{13} = opc; 3023 let Inst{12-10} = Pg; 3024 let Inst{9-5} = Zn; 3025 let Inst{4-0} = Zda; 3026 3027 let Constraints = "$Zda = $_Zda"; 3028 let DestructiveInstType = DestructiveOther; 3029 let ElementSize = zprty.ElementSize; 3030} 3031 3032multiclass sve_int_mlas_vvv_pred<bits<1> opc, string asm, SDPatternOperator op> { 3033 def _B : sve_int_mlas_vvv_pred<0b00, opc, asm, ZPR8>; 3034 def _H : sve_int_mlas_vvv_pred<0b01, opc, asm, ZPR16>; 3035 def _S : sve_int_mlas_vvv_pred<0b10, opc, asm, ZPR32>; 3036 def _D : sve_int_mlas_vvv_pred<0b11, opc, asm, ZPR64>; 3037 3038 def : SVE_4_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3039 def : SVE_4_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3040 def : SVE_4_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3041 def : SVE_4_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3042} 3043 3044//===----------------------------------------------------------------------===// 3045// SVE2 Integer Multiply-Add - Unpredicated Group 3046//===----------------------------------------------------------------------===// 3047 3048class sve2_int_mla<bits<2> sz, bits<5> opc, string asm, 3049 ZPRRegOp zprty1, ZPRRegOp zprty2> 3050: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3051 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3052 bits<5> Zda; 3053 bits<5> Zn; 3054 bits<5> Zm; 3055 let Inst{31-24} = 0b01000100; 3056 let Inst{23-22} = sz; 3057 let Inst{21} = 0b0; 3058 let Inst{20-16} = Zm; 3059 let Inst{15} = 0b0; 3060 let Inst{14-10} = opc; 3061 let Inst{9-5} = Zn; 3062 let Inst{4-0} = Zda; 3063 3064 let Constraints = "$Zda = $_Zda"; 3065 let DestructiveInstType = DestructiveOther; 3066 let ElementSize = ElementSizeNone; 3067} 3068 3069multiclass sve2_int_mla<bit S, string asm, SDPatternOperator op> { 3070 def _B : sve2_int_mla<0b00, { 0b1110, S }, asm, ZPR8, ZPR8>; 3071 def _H : sve2_int_mla<0b01, { 0b1110, S }, asm, ZPR16, ZPR16>; 3072 def _S : sve2_int_mla<0b10, { 0b1110, S }, asm, ZPR32, ZPR32>; 3073 def _D : sve2_int_mla<0b11, { 0b1110, S }, asm, ZPR64, ZPR64>; 3074 3075 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3076 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3077 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3078 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3079} 3080 3081multiclass sve2_int_mla_long<bits<5> opc, string asm, SDPatternOperator op> { 3082 def _H : sve2_int_mla<0b01, opc, asm, ZPR16, ZPR8>; 3083 def _S : sve2_int_mla<0b10, opc, asm, ZPR32, ZPR16>; 3084 def _D : sve2_int_mla<0b11, opc, asm, ZPR64, ZPR32>; 3085 3086 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3087 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3088 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3089} 3090 3091//===----------------------------------------------------------------------===// 3092// SVE2 Integer Multiply-Add - Indexed Group 3093//===----------------------------------------------------------------------===// 3094 3095class sve2_int_mla_by_indexed_elem<bits<2> sz, bits<6> opc, string asm, 3096 ZPRRegOp zprty1, ZPRRegOp zprty2, 3097 ZPRRegOp zprty3, Operand itype> 3098: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3099 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 3100 bits<5> Zda; 3101 bits<5> Zn; 3102 let Inst{31-24} = 0b01000100; 3103 let Inst{23-22} = sz; 3104 let Inst{21} = 0b1; 3105 let Inst{15-10} = opc; 3106 let Inst{9-5} = Zn; 3107 let Inst{4-0} = Zda; 3108 3109 let Constraints = "$Zda = $_Zda"; 3110 let DestructiveInstType = DestructiveOther; 3111 let ElementSize = ElementSizeNone; 3112} 3113 3114multiclass sve2_int_mla_by_indexed_elem<bits<2> opc, bit S, string asm, 3115 SDPatternOperator op> { 3116 def _H : sve2_int_mla_by_indexed_elem<{0, ?}, { 0b000, opc, S }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3117 bits<3> Zm; 3118 bits<3> iop; 3119 let Inst{22} = iop{2}; 3120 let Inst{20-19} = iop{1-0}; 3121 let Inst{18-16} = Zm; 3122 } 3123 def _S : sve2_int_mla_by_indexed_elem<0b10, { 0b000, opc, S }, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3124 bits<3> Zm; 3125 bits<2> iop; 3126 let Inst{20-19} = iop; 3127 let Inst{18-16} = Zm; 3128 } 3129 def _D : sve2_int_mla_by_indexed_elem<0b11, { 0b000, opc, S }, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3130 bits<4> Zm; 3131 bit iop; 3132 let Inst{20} = iop; 3133 let Inst{19-16} = Zm; 3134 } 3135 3136 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3137 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3138 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3139} 3140 3141//===----------------------------------------------------------------------===// 3142// SVE2 Integer Multiply-Add Long - Indexed Group 3143//===----------------------------------------------------------------------===// 3144 3145multiclass sve2_int_mla_long_by_indexed_elem<bits<4> opc, string asm, 3146 SDPatternOperator op> { 3147 def _S : sve2_int_mla_by_indexed_elem<0b10, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3148 asm, ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3149 bits<3> Zm; 3150 bits<3> iop; 3151 let Inst{20-19} = iop{2-1}; 3152 let Inst{18-16} = Zm; 3153 let Inst{11} = iop{0}; 3154 } 3155 def _D : sve2_int_mla_by_indexed_elem<0b11, { opc{3}, 0b0, opc{2-1}, ?, opc{0} }, 3156 asm, ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3157 bits<4> Zm; 3158 bits<2> iop; 3159 let Inst{20} = iop{1}; 3160 let Inst{19-16} = Zm; 3161 let Inst{11} = iop{0}; 3162 } 3163 3164 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3165 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3166} 3167 3168//===----------------------------------------------------------------------===// 3169// SVE Integer Dot Product Group 3170//===----------------------------------------------------------------------===// 3171 3172class sve_intx_dot<bit sz, bit U, string asm, ZPRRegOp zprty1, 3173 ZPRRegOp zprty2> 3174: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), asm, 3175 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3176 bits<5> Zda; 3177 bits<5> Zn; 3178 bits<5> Zm; 3179 let Inst{31-23} = 0b010001001; 3180 let Inst{22} = sz; 3181 let Inst{21} = 0; 3182 let Inst{20-16} = Zm; 3183 let Inst{15-11} = 0; 3184 let Inst{10} = U; 3185 let Inst{9-5} = Zn; 3186 let Inst{4-0} = Zda; 3187 3188 let Constraints = "$Zda = $_Zda"; 3189 let DestructiveInstType = DestructiveOther; 3190} 3191 3192multiclass sve_intx_dot<bit opc, string asm, SDPatternOperator op> { 3193 def _S : sve_intx_dot<0b0, opc, asm, ZPR32, ZPR8>; 3194 def _D : sve_intx_dot<0b1, opc, asm, ZPR64, ZPR16>; 3195 3196 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _S)>; 3197 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _D)>; 3198} 3199 3200//===----------------------------------------------------------------------===// 3201// SVE Integer Dot Product Group - Indexed Group 3202//===----------------------------------------------------------------------===// 3203 3204class sve_intx_dot_by_indexed_elem<bit sz, bit U, string asm, 3205 ZPRRegOp zprty1, ZPRRegOp zprty2, 3206 ZPRRegOp zprty3, Operand itype> 3207: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop), 3208 asm, "\t$Zda, $Zn, $Zm$iop", 3209 "", []>, Sched<[]> { 3210 bits<5> Zda; 3211 bits<5> Zn; 3212 let Inst{31-23} = 0b010001001; 3213 let Inst{22} = sz; 3214 let Inst{21} = 0b1; 3215 let Inst{15-11} = 0; 3216 let Inst{10} = U; 3217 let Inst{9-5} = Zn; 3218 let Inst{4-0} = Zda; 3219 3220 let Constraints = "$Zda = $_Zda"; 3221 let DestructiveInstType = DestructiveOther; 3222} 3223 3224multiclass sve_intx_dot_by_indexed_elem<bit opc, string asm, 3225 SDPatternOperator op> { 3226 def _S : sve_intx_dot_by_indexed_elem<0b0, opc, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b_timm> { 3227 bits<2> iop; 3228 bits<3> Zm; 3229 let Inst{20-19} = iop; 3230 let Inst{18-16} = Zm; 3231 } 3232 def _D : sve_intx_dot_by_indexed_elem<0b1, opc, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b_timm> { 3233 bits<1> iop; 3234 bits<4> Zm; 3235 let Inst{20} = iop; 3236 let Inst{19-16} = Zm; 3237 } 3238 3239 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3240 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv8i16, nxv8i16, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3241} 3242 3243//===----------------------------------------------------------------------===// 3244// SVE2 Complex Integer Dot Product Group 3245//===----------------------------------------------------------------------===// 3246 3247class sve2_complex_int_arith<bits<2> sz, bits<4> opc, string asm, 3248 ZPRRegOp zprty1, ZPRRegOp zprty2> 3249: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm, 3250 complexrotateop:$rot), 3251 asm, "\t$Zda, $Zn, $Zm, $rot", "", []>, Sched<[]> { 3252 bits<5> Zda; 3253 bits<5> Zn; 3254 bits<5> Zm; 3255 bits<2> rot; 3256 let Inst{31-24} = 0b01000100; 3257 let Inst{23-22} = sz; 3258 let Inst{21} = 0b0; 3259 let Inst{20-16} = Zm; 3260 let Inst{15-12} = opc; 3261 let Inst{11-10} = rot; 3262 let Inst{9-5} = Zn; 3263 let Inst{4-0} = Zda; 3264 3265 let Constraints = "$Zda = $_Zda"; 3266 let DestructiveInstType = DestructiveOther; 3267 let ElementSize = ElementSizeNone; 3268} 3269 3270multiclass sve2_cintx_dot<string asm, SDPatternOperator op> { 3271 def _S : sve2_complex_int_arith<0b10, 0b0001, asm, ZPR32, ZPR8>; 3272 def _D : sve2_complex_int_arith<0b11, 0b0001, asm, ZPR64, ZPR16>; 3273 3274 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3275 (i32 complexrotateop:$imm))), 3276 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, complexrotateop:$imm)>; 3277 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3278 (i32 complexrotateop:$imm))), 3279 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, complexrotateop:$imm)>; 3280} 3281 3282//===----------------------------------------------------------------------===// 3283// SVE2 Complex Multiply-Add Group 3284//===----------------------------------------------------------------------===// 3285 3286multiclass sve2_int_cmla<bit opc, string asm, SDPatternOperator op> { 3287 def _B : sve2_complex_int_arith<0b00, { 0b001, opc }, asm, ZPR8, ZPR8>; 3288 def _H : sve2_complex_int_arith<0b01, { 0b001, opc }, asm, ZPR16, ZPR16>; 3289 def _S : sve2_complex_int_arith<0b10, { 0b001, opc }, asm, ZPR32, ZPR32>; 3290 def _D : sve2_complex_int_arith<0b11, { 0b001, opc }, asm, ZPR64, ZPR64>; 3291 3292 def : SVE_4_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, i32, complexrotateop, !cast<Instruction>(NAME # _B)>; 3293 def : SVE_4_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, i32, complexrotateop, !cast<Instruction>(NAME # _H)>; 3294 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, i32, complexrotateop, !cast<Instruction>(NAME # _S)>; 3295 def : SVE_4_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, i32, complexrotateop, !cast<Instruction>(NAME # _D)>; 3296} 3297 3298//===----------------------------------------------------------------------===// 3299// SVE2 Complex Integer Dot Product - Indexed Group 3300//===----------------------------------------------------------------------===// 3301 3302class sve2_complex_int_arith_indexed<bits<2> sz, bits<4> opc, string asm, 3303 ZPRRegOp zprty1, ZPRRegOp zprty2, 3304 ZPRRegOp zprty3, Operand itype> 3305: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty3:$Zm, itype:$iop, 3306 complexrotateop:$rot), 3307 asm, "\t$Zda, $Zn, $Zm$iop, $rot", "", []>, Sched<[]> { 3308 bits<5> Zda; 3309 bits<5> Zn; 3310 bits<2> rot; 3311 let Inst{31-24} = 0b01000100; 3312 let Inst{23-22} = sz; 3313 let Inst{21} = 0b1; 3314 let Inst{15-12} = opc; 3315 let Inst{11-10} = rot; 3316 let Inst{9-5} = Zn; 3317 let Inst{4-0} = Zda; 3318 3319 let Constraints = "$Zda = $_Zda"; 3320 let DestructiveInstType = DestructiveOther; 3321 let ElementSize = ElementSizeNone; 3322} 3323 3324multiclass sve2_cintx_dot_by_indexed_elem<string asm, SDPatternOperator op> { 3325 def _S : sve2_complex_int_arith_indexed<0b10, 0b0100, asm, ZPR32, ZPR8, ZPR3b8, VectorIndexS32b> { 3326 bits<2> iop; 3327 bits<3> Zm; 3328 let Inst{20-19} = iop; 3329 let Inst{18-16} = Zm; 3330 } 3331 def _D : sve2_complex_int_arith_indexed<0b11, 0b0100, asm, ZPR64, ZPR16, ZPR4b16, VectorIndexD32b> { 3332 bit iop; 3333 bits<4> Zm; 3334 let Inst{20} = iop; 3335 let Inst{19-16} = Zm; 3336 } 3337 3338 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv16i8 ZPR8:$Op2), (nxv16i8 ZPR8:$Op3), 3339 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3340 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR8:$Op2, ZPR8:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3341 def : Pat<(nxv2i64 (op (nxv2i64 ZPR64:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3342 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3343 (!cast<Instruction>(NAME # "_D") ZPR64:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3344} 3345 3346//===----------------------------------------------------------------------===// 3347// SVE2 Complex Multiply-Add - Indexed Group 3348//===----------------------------------------------------------------------===// 3349 3350multiclass sve2_cmla_by_indexed_elem<bit opc, string asm, 3351 SDPatternOperator op> { 3352 def _H : sve2_complex_int_arith_indexed<0b10, { 0b011, opc }, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexS32b> { 3353 bits<2> iop; 3354 bits<3> Zm; 3355 let Inst{20-19} = iop; 3356 let Inst{18-16} = Zm; 3357 } 3358 def _S : sve2_complex_int_arith_indexed<0b11, { 0b011, opc }, asm, ZPR32, ZPR32, ZPR4b32, VectorIndexD32b> { 3359 bit iop; 3360 bits<4> Zm; 3361 let Inst{20} = iop; 3362 let Inst{19-16} = Zm; 3363 } 3364 3365 def : Pat<(nxv8i16 (op (nxv8i16 ZPR16:$Op1), (nxv8i16 ZPR16:$Op2), (nxv8i16 ZPR16:$Op3), 3366 (i32 VectorIndexS32b_timm:$idx), (i32 complexrotateop:$imm))), 3367 (!cast<Instruction>(NAME # "_H") ZPR16:$Op1, ZPR16:$Op2, ZPR16:$Op3, VectorIndexS32b_timm:$idx, complexrotateop:$imm)>; 3368 3369 def : Pat<(nxv4i32 (op (nxv4i32 ZPR32:$Op1), (nxv4i32 ZPR32:$Op2), (nxv4i32 ZPR32:$Op3), 3370 (i32 VectorIndexD32b_timm:$idx), (i32 complexrotateop:$imm))), 3371 (!cast<Instruction>(NAME # "_S") ZPR32:$Op1, ZPR32:$Op2, ZPR32:$Op3, VectorIndexD32b_timm:$idx, complexrotateop:$imm)>; 3372} 3373 3374//===----------------------------------------------------------------------===// 3375// SVE2 Integer Multiply - Unpredicated Group 3376//===----------------------------------------------------------------------===// 3377 3378class sve2_int_mul<bits<2> sz, bits<3> opc, string asm, ZPRRegOp zprty> 3379: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 3380 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3381 bits<5> Zd; 3382 bits<5> Zm; 3383 bits<5> Zn; 3384 let Inst{31-24} = 0b00000100; 3385 let Inst{23-22} = sz; 3386 let Inst{21} = 0b1; 3387 let Inst{20-16} = Zm; 3388 let Inst{15-13} = 0b011; 3389 let Inst{12-10} = opc; 3390 let Inst{9-5} = Zn; 3391 let Inst{4-0} = Zd; 3392} 3393 3394multiclass sve2_int_mul<bits<3> opc, string asm, SDPatternOperator op, 3395 SDPatternOperator op_pred = null_frag> { 3396 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3397 def _H : sve2_int_mul<0b01, opc, asm, ZPR16>; 3398 def _S : sve2_int_mul<0b10, opc, asm, ZPR32>; 3399 def _D : sve2_int_mul<0b11, opc, asm, ZPR64>; 3400 3401 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3402 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3403 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3404 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3405 3406 def : SVE_2_Op_Pred_All_Active<nxv16i8, op_pred, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3407 def : SVE_2_Op_Pred_All_Active<nxv8i16, op_pred, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3408 def : SVE_2_Op_Pred_All_Active<nxv4i32, op_pred, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3409 def : SVE_2_Op_Pred_All_Active<nxv2i64, op_pred, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3410} 3411 3412multiclass sve2_int_mul_single<bits<3> opc, string asm, SDPatternOperator op> { 3413 def _B : sve2_int_mul<0b00, opc, asm, ZPR8>; 3414 3415 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3416} 3417 3418//===----------------------------------------------------------------------===// 3419// SVE2 Integer Multiply - Indexed Group 3420//===----------------------------------------------------------------------===// 3421 3422class sve2_int_mul_by_indexed_elem<bits<2> sz, bits<4> opc, string asm, 3423 ZPRRegOp zprty1, ZPRRegOp zprty2, 3424 ZPRRegOp zprty3, Operand itype> 3425: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm, itype:$iop), 3426 asm, "\t$Zd, $Zn, $Zm$iop", "", []>, Sched<[]> { 3427 bits<5> Zd; 3428 bits<5> Zn; 3429 let Inst{31-24} = 0b01000100; 3430 let Inst{23-22} = sz; 3431 let Inst{21} = 0b1; 3432 let Inst{15-14} = 0b11; 3433 let Inst{13-10} = opc; 3434 let Inst{9-5} = Zn; 3435 let Inst{4-0} = Zd; 3436} 3437 3438multiclass sve2_int_mul_by_indexed_elem<bits<4> opc, string asm, 3439 SDPatternOperator op> { 3440 def _H : sve2_int_mul_by_indexed_elem<{0, ?}, opc, asm, ZPR16, ZPR16, ZPR3b16, VectorIndexH32b> { 3441 bits<3> Zm; 3442 bits<3> iop; 3443 let Inst{22} = iop{2}; 3444 let Inst{20-19} = iop{1-0}; 3445 let Inst{18-16} = Zm; 3446 } 3447 def _S : sve2_int_mul_by_indexed_elem<0b10, opc, asm, ZPR32, ZPR32, ZPR3b32, VectorIndexS32b> { 3448 bits<3> Zm; 3449 bits<2> iop; 3450 let Inst{20-19} = iop; 3451 let Inst{18-16} = Zm; 3452 } 3453 def _D : sve2_int_mul_by_indexed_elem<0b11, opc, asm, ZPR64, ZPR64, ZPR4b64, VectorIndexD32b> { 3454 bits<4> Zm; 3455 bit iop; 3456 let Inst{20} = iop; 3457 let Inst{19-16} = Zm; 3458 } 3459 3460 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _H)>; 3461 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _S)>; 3462 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, VectorIndexD32b_timm, !cast<Instruction>(NAME # _D)>; 3463} 3464 3465multiclass sve2_int_mul_long_by_indexed_elem<bits<3> opc, string asm, 3466 SDPatternOperator op> { 3467 def _S : sve2_int_mul_by_indexed_elem<0b10, { opc{2-1}, ?, opc{0} }, asm, 3468 ZPR32, ZPR16, ZPR3b16, VectorIndexH32b> { 3469 bits<3> Zm; 3470 bits<3> iop; 3471 let Inst{20-19} = iop{2-1}; 3472 let Inst{18-16} = Zm; 3473 let Inst{11} = iop{0}; 3474 } 3475 def _D : sve2_int_mul_by_indexed_elem<0b11, { opc{2-1}, ?, opc{0} }, asm, 3476 ZPR64, ZPR32, ZPR4b32, VectorIndexS32b> { 3477 bits<4> Zm; 3478 bits<2> iop; 3479 let Inst{20} = iop{1}; 3480 let Inst{19-16} = Zm; 3481 let Inst{11} = iop{0}; 3482 } 3483 3484 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv8i16, nxv8i16, i32, VectorIndexH32b_timm, !cast<Instruction>(NAME # _S)>; 3485 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv4i32, nxv4i32, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME # _D)>; 3486} 3487 3488//===----------------------------------------------------------------------===// 3489// SVE2 Integer - Predicated Group 3490//===----------------------------------------------------------------------===// 3491 3492class sve2_int_arith_pred<bits<2> sz, bits<6> opc, string asm, 3493 ZPRRegOp zprty> 3494: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 3495 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", "", []>, Sched<[]> { 3496 bits<3> Pg; 3497 bits<5> Zm; 3498 bits<5> Zdn; 3499 let Inst{31-24} = 0b01000100; 3500 let Inst{23-22} = sz; 3501 let Inst{21-20} = 0b01; 3502 let Inst{20-16} = opc{5-1}; 3503 let Inst{15-14} = 0b10; 3504 let Inst{13} = opc{0}; 3505 let Inst{12-10} = Pg; 3506 let Inst{9-5} = Zm; 3507 let Inst{4-0} = Zdn; 3508 3509 let Constraints = "$Zdn = $_Zdn"; 3510 let DestructiveInstType = DestructiveOther; 3511 let ElementSize = zprty.ElementSize; 3512} 3513 3514multiclass sve2_int_arith_pred<bits<6> opc, string asm, SDPatternOperator op, 3515 string Ps = "", 3516 DestructiveInstTypeEnum flags=DestructiveOther, 3517 string revname="", bit isReverseInstr=0> { 3518 let DestructiveInstType = flags in { 3519 def _B : sve2_int_arith_pred<0b00, opc, asm, ZPR8>, 3520 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 3521 def _H : sve2_int_arith_pred<0b01, opc, asm, ZPR16>, 3522 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 3523 def _S : sve2_int_arith_pred<0b10, opc, asm, ZPR32>, 3524 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 3525 def _D : sve2_int_arith_pred<0b11, opc, asm, ZPR64>, 3526 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 3527 } 3528 3529 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3530 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3531 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3532 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3533} 3534 3535class sve2_int_sadd_long_accum_pairwise<bits<2> sz, bit U, string asm, 3536 ZPRRegOp zprty1, ZPRRegOp zprty2> 3537: I<(outs zprty1:$Zda), (ins PPR3bAny:$Pg, zprty1:$_Zda, zprty2:$Zn), 3538 asm, "\t$Zda, $Pg/m, $Zn", "", []>, Sched<[]> { 3539 bits<3> Pg; 3540 bits<5> Zn; 3541 bits<5> Zda; 3542 let Inst{31-24} = 0b01000100; 3543 let Inst{23-22} = sz; 3544 let Inst{21-17} = 0b00010; 3545 let Inst{16} = U; 3546 let Inst{15-13} = 0b101; 3547 let Inst{12-10} = Pg; 3548 let Inst{9-5} = Zn; 3549 let Inst{4-0} = Zda; 3550 3551 let Constraints = "$Zda = $_Zda"; 3552 let DestructiveInstType = DestructiveOther; 3553 let ElementSize = zprty1.ElementSize; 3554} 3555 3556multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperator op> { 3557 def _H : sve2_int_sadd_long_accum_pairwise<0b01, U, asm, ZPR16, ZPR8>; 3558 def _S : sve2_int_sadd_long_accum_pairwise<0b10, U, asm, ZPR32, ZPR16>; 3559 def _D : sve2_int_sadd_long_accum_pairwise<0b11, U, asm, ZPR64, ZPR32>; 3560 3561 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3562 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3563 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3564} 3565 3566class sve2_int_un_pred_arit<bits<2> sz, bit Q, bits<2> opc, 3567 string asm, ZPRRegOp zprty> 3568: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 3569 asm, "\t$Zd, $Pg/m, $Zn", 3570 "", 3571 []>, Sched<[]> { 3572 bits<3> Pg; 3573 bits<5> Zd; 3574 bits<5> Zn; 3575 let Inst{31-24} = 0b01000100; 3576 let Inst{23-22} = sz; 3577 let Inst{21-20} = 0b00; 3578 let Inst{19} = Q; 3579 let Inst{18} = 0b0; 3580 let Inst{17-16} = opc; 3581 let Inst{15-13} = 0b101; 3582 let Inst{12-10} = Pg; 3583 let Inst{9-5} = Zn; 3584 let Inst{4-0} = Zd; 3585 3586 let Constraints = "$Zd = $_Zd"; 3587 let DestructiveInstType = DestructiveUnaryPassthru; 3588 let ElementSize = zprty.ElementSize; 3589} 3590 3591multiclass sve2_int_un_pred_arit_s<bits<3> opc, string asm, 3592 SDPatternOperator op> { 3593 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3594 SVEPseudo2Instr<NAME # _S, 1>; 3595 3596 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3597 3598 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3599 3600 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3601} 3602 3603multiclass sve2_int_un_pred_arit<bits<3> opc, string asm, SDPatternOperator op> { 3604 def _B : sve2_int_un_pred_arit<0b00, opc{2}, opc{1-0}, asm, ZPR8>, 3605 SVEPseudo2Instr<NAME # _B, 1>; 3606 def _H : sve2_int_un_pred_arit<0b01, opc{2}, opc{1-0}, asm, ZPR16>, 3607 SVEPseudo2Instr<NAME # _H, 1>; 3608 def _S : sve2_int_un_pred_arit<0b10, opc{2}, opc{1-0}, asm, ZPR32>, 3609 SVEPseudo2Instr<NAME # _S, 1>; 3610 def _D : sve2_int_un_pred_arit<0b11, opc{2}, opc{1-0}, asm, ZPR64>, 3611 SVEPseudo2Instr<NAME # _D, 1>; 3612 3613 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 3614 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 3615 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 3616 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 3617 3618 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 3619 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 3620 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 3621 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 3622 3623 defm : SVE_3_Op_Undef_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 3624 defm : SVE_3_Op_Undef_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 3625 defm : SVE_3_Op_Undef_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 3626 defm : SVE_3_Op_Undef_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 3627} 3628 3629//===----------------------------------------------------------------------===// 3630// SVE2 Widening Integer Arithmetic Group 3631//===----------------------------------------------------------------------===// 3632 3633class sve2_wide_int_arith<bits<2> sz, bits<5> opc, string asm, 3634 ZPRRegOp zprty1, ZPRRegOp zprty2, ZPRRegOp zprty3> 3635: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty3:$Zm), 3636 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3637 bits<5> Zd; 3638 bits<5> Zn; 3639 bits<5> Zm; 3640 let Inst{31-24} = 0b01000101; 3641 let Inst{23-22} = sz; 3642 let Inst{21} = 0b0; 3643 let Inst{20-16} = Zm; 3644 let Inst{15} = 0b0; 3645 let Inst{14-10} = opc; 3646 let Inst{9-5} = Zn; 3647 let Inst{4-0} = Zd; 3648} 3649 3650multiclass sve2_wide_int_arith_long<bits<5> opc, string asm, 3651 SDPatternOperator op> { 3652 def _H : sve2_wide_int_arith<0b01, opc, asm, ZPR16, ZPR8, ZPR8>; 3653 def _S : sve2_wide_int_arith<0b10, opc, asm, ZPR32, ZPR16, ZPR16>; 3654 def _D : sve2_wide_int_arith<0b11, opc, asm, ZPR64, ZPR32, ZPR32>; 3655 3656 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3657 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3658 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3659} 3660 3661multiclass sve2_wide_int_arith_wide<bits<3> opc, string asm, 3662 SDPatternOperator op> { 3663 def _H : sve2_wide_int_arith<0b01, { 0b10, opc }, asm, ZPR16, ZPR16, ZPR8>; 3664 def _S : sve2_wide_int_arith<0b10, { 0b10, opc }, asm, ZPR32, ZPR32, ZPR16>; 3665 def _D : sve2_wide_int_arith<0b11, { 0b10, opc }, asm, ZPR64, ZPR64, ZPR32>; 3666 3667 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>; 3668 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>; 3669 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>; 3670} 3671 3672multiclass sve2_wide_int_arith_pmul<bits<2> sz, bits<5> opc, string asm, 3673 SDPatternOperator op> { 3674 def NAME : sve2_wide_int_arith<sz, opc, asm, ZPR128, ZPR64, ZPR64>; 3675 3676 // To avoid using 128 bit elements in the IR, the pattern below works with 3677 // llvm intrinsics with the _pair suffix, to reflect that 3678 // _Q is implemented as a pair of _D. 3679 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 3680} 3681 3682multiclass sve2_pmul_long<bits<1> opc, string asm, SDPatternOperator op> { 3683 def _H : sve2_wide_int_arith<0b01, {0b1101, opc}, asm, ZPR16, ZPR8, ZPR8>; 3684 def _D : sve2_wide_int_arith<0b11, {0b1101, opc}, asm, ZPR64, ZPR32, ZPR32>; 3685 3686 // To avoid using 128 bit elements in the IR, the patterns below work with 3687 // llvm intrinsics with the _pair suffix, to reflect that 3688 // _H is implemented as a pair of _B and _D is implemented as a pair of _S. 3689 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3690 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3691} 3692 3693//===----------------------------------------------------------------------===// 3694// SVE2 Misc Group 3695//===----------------------------------------------------------------------===// 3696 3697class sve2_misc<bits<2> sz, bits<4> opc, string asm, 3698 ZPRRegOp zprty1, ZPRRegOp zprty2> 3699: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 3700 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3701 bits<5> Zd; 3702 bits<5> Zn; 3703 bits<5> Zm; 3704 let Inst{31-24} = 0b01000101; 3705 let Inst{23-22} = sz; 3706 let Inst{21} = 0b0; 3707 let Inst{20-16} = Zm; 3708 let Inst{15-14} = 0b10; 3709 let Inst{13-10} = opc; 3710 let Inst{9-5} = Zn; 3711 let Inst{4-0} = Zd; 3712} 3713 3714multiclass sve2_misc_bitwise<bits<4> opc, string asm, SDPatternOperator op> { 3715 def _B : sve2_misc<0b00, opc, asm, ZPR8, ZPR8>; 3716 def _H : sve2_misc<0b01, opc, asm, ZPR16, ZPR16>; 3717 def _S : sve2_misc<0b10, opc, asm, ZPR32, ZPR32>; 3718 def _D : sve2_misc<0b11, opc, asm, ZPR64, ZPR64>; 3719 3720 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3721 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3722 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3723 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3724} 3725 3726multiclass sve2_misc_int_addsub_long_interleaved<bits<2> opc, string asm, 3727 SDPatternOperator op> { 3728 def _H : sve2_misc<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3729 def _S : sve2_misc<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3730 def _D : sve2_misc<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3731 3732 def : SVE_2_Op_Pat<nxv8i16, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3733 def : SVE_2_Op_Pat<nxv4i32, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3734 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3735} 3736 3737class sve2_bitwise_xor_interleaved<bits<2> sz, bits<1> opc, string asm, 3738 ZPRRegOp zprty1, ZPRRegOp zprty2> 3739: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 3740 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 3741 bits<5> Zd; 3742 bits<5> Zn; 3743 bits<5> Zm; 3744 let Inst{31-24} = 0b01000101; 3745 let Inst{23-22} = sz; 3746 let Inst{21} = 0b0; 3747 let Inst{20-16} = Zm; 3748 let Inst{15-11} = 0b10010; 3749 let Inst{10} = opc; 3750 let Inst{9-5} = Zn; 3751 let Inst{4-0} = Zd; 3752 3753 let Constraints = "$Zd = $_Zd"; 3754 let DestructiveInstType = DestructiveOther; 3755 let ElementSize = ElementSizeNone; 3756} 3757 3758multiclass sve2_bitwise_xor_interleaved<bit opc, string asm, 3759 SDPatternOperator op> { 3760 def _B : sve2_bitwise_xor_interleaved<0b00, opc, asm, ZPR8, ZPR8>; 3761 def _H : sve2_bitwise_xor_interleaved<0b01, opc, asm, ZPR16, ZPR16>; 3762 def _S : sve2_bitwise_xor_interleaved<0b10, opc, asm, ZPR32, ZPR32>; 3763 def _D : sve2_bitwise_xor_interleaved<0b11, opc, asm, ZPR64, ZPR64>; 3764 3765 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3766 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3767 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3768 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3769} 3770 3771class sve2_bitwise_shift_left_long<bits<3> tsz8_64, bits<2> opc, string asm, 3772 ZPRRegOp zprty1, ZPRRegOp zprty2, 3773 Operand immtype> 3774: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 3775 asm, "\t$Zd, $Zn, $imm", 3776 "", []>, Sched<[]> { 3777 bits<5> Zd; 3778 bits<5> Zn; 3779 bits<5> imm; 3780 let Inst{31-23} = 0b010001010; 3781 let Inst{22} = tsz8_64{2}; 3782 let Inst{21} = 0b0; 3783 let Inst{20-19} = tsz8_64{1-0}; 3784 let Inst{18-16} = imm{2-0}; // imm3 3785 let Inst{15-12} = 0b1010; 3786 let Inst{11-10} = opc; 3787 let Inst{9-5} = Zn; 3788 let Inst{4-0} = Zd; 3789} 3790 3791multiclass sve2_bitwise_shift_left_long<bits<2> opc, string asm, 3792 SDPatternOperator op> { 3793 def _H : sve2_bitwise_shift_left_long<{0,0,1}, opc, asm, 3794 ZPR16, ZPR8, vecshiftL8>; 3795 def _S : sve2_bitwise_shift_left_long<{0,1,?}, opc, asm, 3796 ZPR32, ZPR16, vecshiftL16> { 3797 let Inst{19} = imm{3}; 3798 } 3799 def _D : sve2_bitwise_shift_left_long<{1,?,?}, opc, asm, 3800 ZPR64, ZPR32, vecshiftL32> { 3801 let Inst{20-19} = imm{4-3}; 3802 } 3803 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _H)>; 3804 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _S)>; 3805 def : SVE_2_Op_Imm_Pat<nxv2i64, op, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _D)>; 3806} 3807 3808//===----------------------------------------------------------------------===// 3809// SVE2 Accumulate Group 3810//===----------------------------------------------------------------------===// 3811 3812class sve2_int_bin_shift_imm<bits<4> tsz8_64, bit opc, string asm, 3813 ZPRRegOp zprty, Operand immtype> 3814: I<(outs zprty:$Zd), (ins zprty:$_Zd, zprty:$Zn, immtype:$imm), 3815 asm, "\t$Zd, $Zn, $imm", 3816 "", []>, Sched<[]> { 3817 bits<5> Zd; 3818 bits<5> Zn; 3819 bits<6> imm; 3820 let Inst{31-24} = 0b01000101; 3821 let Inst{23-22} = tsz8_64{3-2}; 3822 let Inst{21} = 0b0; 3823 let Inst{20-19} = tsz8_64{1-0}; 3824 let Inst{18-16} = imm{2-0}; // imm3 3825 let Inst{15-11} = 0b11110; 3826 let Inst{10} = opc; 3827 let Inst{9-5} = Zn; 3828 let Inst{4-0} = Zd; 3829 3830 let Constraints = "$Zd = $_Zd"; 3831} 3832 3833multiclass sve2_int_bin_shift_imm_left<bit opc, string asm, 3834 SDPatternOperator op> { 3835 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 3836 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 3837 let Inst{19} = imm{3}; 3838 } 3839 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 3840 let Inst{20-19} = imm{4-3}; 3841 } 3842 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 3843 let Inst{22} = imm{5}; 3844 let Inst{20-19} = imm{4-3}; 3845 } 3846 3847 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 3848 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 3849 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 3850 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 3851} 3852 3853multiclass sve2_int_bin_shift_imm_right<bit opc, string asm, 3854 SDPatternOperator op> { 3855 def _B : sve2_int_bin_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3856 def _H : sve2_int_bin_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3857 let Inst{19} = imm{3}; 3858 } 3859 def _S : sve2_int_bin_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3860 let Inst{20-19} = imm{4-3}; 3861 } 3862 def _D : sve2_int_bin_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3863 let Inst{22} = imm{5}; 3864 let Inst{20-19} = imm{4-3}; 3865 } 3866 3867 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3868 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3869 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3870 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3871} 3872 3873class sve2_int_bin_accum_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 3874 ZPRRegOp zprty, Operand immtype> 3875: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, immtype:$imm), 3876 asm, "\t$Zda, $Zn, $imm", 3877 "", []>, Sched<[]> { 3878 bits<5> Zda; 3879 bits<5> Zn; 3880 bits<6> imm; 3881 let Inst{31-24} = 0b01000101; 3882 let Inst{23-22} = tsz8_64{3-2}; 3883 let Inst{21} = 0b0; 3884 let Inst{20-19} = tsz8_64{1-0}; 3885 let Inst{18-16} = imm{2-0}; // imm3 3886 let Inst{15-12} = 0b1110; 3887 let Inst{11-10} = opc; 3888 let Inst{9-5} = Zn; 3889 let Inst{4-0} = Zda; 3890 3891 let Constraints = "$Zda = $_Zda"; 3892 let DestructiveInstType = DestructiveOther; 3893 let ElementSize = ElementSizeNone; 3894} 3895 3896multiclass sve2_int_bin_accum_shift_imm_right<bits<2> opc, string asm, 3897 SDPatternOperator op, 3898 SDPatternOperator shift_op = null_frag> { 3899 def _B : sve2_int_bin_accum_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 3900 def _H : sve2_int_bin_accum_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 3901 let Inst{19} = imm{3}; 3902 } 3903 def _S : sve2_int_bin_accum_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 3904 let Inst{20-19} = imm{4-3}; 3905 } 3906 def _D : sve2_int_bin_accum_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 3907 let Inst{22} = imm{5}; 3908 let Inst{20-19} = imm{4-3}; 3909 } 3910 3911 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 3912 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 3913 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 3914 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 3915 3916 def : SVE_Shift_Add_All_Active_Pat<nxv16i8, shift_op, nxv16i1, nxv16i8, nxv16i8, i32, !cast<Instruction>(NAME # _B)>; 3917 def : SVE_Shift_Add_All_Active_Pat<nxv8i16, shift_op, nxv8i1, nxv8i16, nxv8i16, i32, !cast<Instruction>(NAME # _H)>; 3918 def : SVE_Shift_Add_All_Active_Pat<nxv4i32, shift_op, nxv4i1, nxv4i32, nxv4i32, i32, !cast<Instruction>(NAME # _S)>; 3919 def : SVE_Shift_Add_All_Active_Pat<nxv2i64, shift_op, nxv2i1, nxv2i64, nxv2i64, i32, !cast<Instruction>(NAME # _D)>; 3920} 3921 3922class sve2_int_cadd<bits<2> sz, bit opc, string asm, ZPRRegOp zprty> 3923: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, complexrotateopodd:$rot), 3924 asm, "\t$Zdn, $_Zdn, $Zm, $rot", "", []>, Sched<[]> { 3925 bits<5> Zdn; 3926 bits<5> Zm; 3927 bit rot; 3928 let Inst{31-24} = 0b01000101; 3929 let Inst{23-22} = sz; 3930 let Inst{21-17} = 0b00000; 3931 let Inst{16} = opc; 3932 let Inst{15-11} = 0b11011; 3933 let Inst{10} = rot; 3934 let Inst{9-5} = Zm; 3935 let Inst{4-0} = Zdn; 3936 3937 let Constraints = "$Zdn = $_Zdn"; 3938 let DestructiveInstType = DestructiveOther; 3939 let ElementSize = ElementSizeNone; 3940} 3941 3942multiclass sve2_int_cadd<bit opc, string asm, SDPatternOperator op> { 3943 def _B : sve2_int_cadd<0b00, opc, asm, ZPR8>; 3944 def _H : sve2_int_cadd<0b01, opc, asm, ZPR16>; 3945 def _S : sve2_int_cadd<0b10, opc, asm, ZPR32>; 3946 def _D : sve2_int_cadd<0b11, opc, asm, ZPR64>; 3947 3948 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, complexrotateopodd, !cast<Instruction>(NAME # _B)>; 3949 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, complexrotateopodd, !cast<Instruction>(NAME # _H)>; 3950 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, complexrotateopodd, !cast<Instruction>(NAME # _S)>; 3951 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, complexrotateopodd, !cast<Instruction>(NAME # _D)>; 3952} 3953 3954class sve2_int_absdiff_accum<bits<2> sz, bits<4> opc, string asm, 3955 ZPRRegOp zprty1, ZPRRegOp zprty2> 3956: I<(outs zprty1:$Zda), (ins zprty1:$_Zda, zprty2:$Zn, zprty2:$Zm), 3957 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 3958 bits<5> Zda; 3959 bits<5> Zn; 3960 bits<5> Zm; 3961 let Inst{31-24} = 0b01000101; 3962 let Inst{23-22} = sz; 3963 let Inst{21} = 0b0; 3964 let Inst{20-16} = Zm; 3965 let Inst{15-14} = 0b11; 3966 let Inst{13-10} = opc; 3967 let Inst{9-5} = Zn; 3968 let Inst{4-0} = Zda; 3969 3970 let Constraints = "$Zda = $_Zda"; 3971 let DestructiveInstType = DestructiveOther; 3972 let ElementSize = ElementSizeNone; 3973} 3974 3975multiclass sve2_int_absdiff_accum<bit opc, string asm, SDPatternOperator op> { 3976 def _B : sve2_int_absdiff_accum<0b00, { 0b111, opc }, asm, ZPR8, ZPR8>; 3977 def _H : sve2_int_absdiff_accum<0b01, { 0b111, opc }, asm, ZPR16, ZPR16>; 3978 def _S : sve2_int_absdiff_accum<0b10, { 0b111, opc }, asm, ZPR32, ZPR32>; 3979 def _D : sve2_int_absdiff_accum<0b11, { 0b111, opc }, asm, ZPR64, ZPR64>; 3980 3981 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 3982 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 3983 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 3984 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 3985} 3986 3987multiclass sve2_int_absdiff_accum_long<bits<2> opc, string asm, 3988 SDPatternOperator op> { 3989 def _H : sve2_int_absdiff_accum<0b01, { 0b00, opc }, asm, ZPR16, ZPR8>; 3990 def _S : sve2_int_absdiff_accum<0b10, { 0b00, opc }, asm, ZPR32, ZPR16>; 3991 def _D : sve2_int_absdiff_accum<0b11, { 0b00, opc }, asm, ZPR64, ZPR32>; 3992 3993 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _H)>; 3994 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _S)>; 3995 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _D)>; 3996} 3997 3998multiclass sve2_int_addsub_long_carry<bits<2> opc, string asm, 3999 SDPatternOperator op> { 4000 def _S : sve2_int_absdiff_accum<{ opc{1}, 0b0 }, { 0b010, opc{0} }, asm, 4001 ZPR32, ZPR32>; 4002 def _D : sve2_int_absdiff_accum<{ opc{1}, 0b1 }, { 0b010, opc{0} }, asm, 4003 ZPR64, ZPR64>; 4004 4005 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 4006 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 4007} 4008 4009//===----------------------------------------------------------------------===// 4010// SVE2 Narrowing Group 4011//===----------------------------------------------------------------------===// 4012 4013class sve2_int_bin_shift_imm_narrow_bottom<bits<3> tsz8_64, bits<3> opc, 4014 string asm, ZPRRegOp zprty1, 4015 ZPRRegOp zprty2, Operand immtype> 4016: I<(outs zprty1:$Zd), (ins zprty2:$Zn, immtype:$imm), 4017 asm, "\t$Zd, $Zn, $imm", 4018 "", []>, Sched<[]> { 4019 bits<5> Zd; 4020 bits<5> Zn; 4021 bits<5> imm; 4022 let Inst{31-23} = 0b010001010; 4023 let Inst{22} = tsz8_64{2}; 4024 let Inst{21} = 0b1; 4025 let Inst{20-19} = tsz8_64{1-0}; 4026 let Inst{18-16} = imm{2-0}; // imm3 4027 let Inst{15-14} = 0b00; 4028 let Inst{13-11} = opc; 4029 let Inst{10} = 0b0; 4030 let Inst{9-5} = Zn; 4031 let Inst{4-0} = Zd; 4032} 4033 4034multiclass sve2_int_bin_shift_imm_right_narrow_bottom<bits<3> opc, string asm, 4035 SDPatternOperator op> { 4036 def _B : sve2_int_bin_shift_imm_narrow_bottom<{0,0,1}, opc, asm, ZPR8, ZPR16, 4037 tvecshiftR8>; 4038 def _H : sve2_int_bin_shift_imm_narrow_bottom<{0,1,?}, opc, asm, ZPR16, ZPR32, 4039 tvecshiftR16> { 4040 let Inst{19} = imm{3}; 4041 } 4042 def _S : sve2_int_bin_shift_imm_narrow_bottom<{1,?,?}, opc, asm, ZPR32, ZPR64, 4043 tvecshiftR32> { 4044 let Inst{20-19} = imm{4-3}; 4045 } 4046 def : SVE_2_Op_Imm_Pat<nxv16i8, op, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4047 def : SVE_2_Op_Imm_Pat<nxv8i16, op, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4048 def : SVE_2_Op_Imm_Pat<nxv4i32, op, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4049} 4050 4051class sve2_int_bin_shift_imm_narrow_top<bits<3> tsz8_64, bits<3> opc, 4052 string asm, ZPRRegOp zprty1, 4053 ZPRRegOp zprty2, Operand immtype> 4054: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, immtype:$imm), 4055 asm, "\t$Zd, $Zn, $imm", 4056 "", []>, Sched<[]> { 4057 bits<5> Zd; 4058 bits<5> Zn; 4059 bits<5> imm; 4060 let Inst{31-23} = 0b010001010; 4061 let Inst{22} = tsz8_64{2}; 4062 let Inst{21} = 0b1; 4063 let Inst{20-19} = tsz8_64{1-0}; 4064 let Inst{18-16} = imm{2-0}; // imm3 4065 let Inst{15-14} = 0b00; 4066 let Inst{13-11} = opc; 4067 let Inst{10} = 0b1; 4068 let Inst{9-5} = Zn; 4069 let Inst{4-0} = Zd; 4070 4071 let Constraints = "$Zd = $_Zd"; 4072} 4073 4074multiclass sve2_int_bin_shift_imm_right_narrow_top<bits<3> opc, string asm, 4075 SDPatternOperator op> { 4076 def _B : sve2_int_bin_shift_imm_narrow_top<{0,0,1}, opc, asm, ZPR8, ZPR16, 4077 tvecshiftR8>; 4078 def _H : sve2_int_bin_shift_imm_narrow_top<{0,1,?}, opc, asm, ZPR16, ZPR32, 4079 tvecshiftR16> { 4080 let Inst{19} = imm{3}; 4081 } 4082 def _S : sve2_int_bin_shift_imm_narrow_top<{1,?,?}, opc, asm, ZPR32, ZPR64, 4083 tvecshiftR32> { 4084 let Inst{20-19} = imm{4-3}; 4085 } 4086 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv8i16, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4087 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv4i32, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4088 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv2i64, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4089} 4090 4091class sve2_int_addsub_narrow_high_bottom<bits<2> sz, bits<2> opc, string asm, 4092 ZPRRegOp zprty1, ZPRRegOp zprty2> 4093: I<(outs zprty1:$Zd), (ins zprty2:$Zn, zprty2:$Zm), 4094 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4095 bits<5> Zd; 4096 bits<5> Zn; 4097 bits<5> Zm; 4098 let Inst{31-24} = 0b01000101; 4099 let Inst{23-22} = sz; 4100 let Inst{21} = 0b1; 4101 let Inst{20-16} = Zm; 4102 let Inst{15-13} = 0b011; 4103 let Inst{12-11} = opc; // S, R 4104 let Inst{10} = 0b0; // Top 4105 let Inst{9-5} = Zn; 4106 let Inst{4-0} = Zd; 4107} 4108 4109multiclass sve2_int_addsub_narrow_high_bottom<bits<2> opc, string asm, 4110 SDPatternOperator op> { 4111 def _B : sve2_int_addsub_narrow_high_bottom<0b01, opc, asm, ZPR8, ZPR16>; 4112 def _H : sve2_int_addsub_narrow_high_bottom<0b10, opc, asm, ZPR16, ZPR32>; 4113 def _S : sve2_int_addsub_narrow_high_bottom<0b11, opc, asm, ZPR32, ZPR64>; 4114 4115 def : SVE_2_Op_Pat<nxv16i8, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4116 def : SVE_2_Op_Pat<nxv8i16, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4117 def : SVE_2_Op_Pat<nxv4i32, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4118} 4119 4120class sve2_int_addsub_narrow_high_top<bits<2> sz, bits<2> opc, string asm, 4121 ZPRRegOp zprty1, ZPRRegOp zprty2> 4122: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn, zprty2:$Zm), 4123 asm, "\t$Zd, $Zn, $Zm", "", []>, Sched<[]> { 4124 bits<5> Zd; 4125 bits<5> Zn; 4126 bits<5> Zm; 4127 let Inst{31-24} = 0b01000101; 4128 let Inst{23-22} = sz; 4129 let Inst{21} = 0b1; 4130 let Inst{20-16} = Zm; 4131 let Inst{15-13} = 0b011; 4132 let Inst{12-11} = opc; // S, R 4133 let Inst{10} = 0b1; // Top 4134 let Inst{9-5} = Zn; 4135 let Inst{4-0} = Zd; 4136 4137 let Constraints = "$Zd = $_Zd"; 4138} 4139 4140multiclass sve2_int_addsub_narrow_high_top<bits<2> opc, string asm, 4141 SDPatternOperator op> { 4142 def _B : sve2_int_addsub_narrow_high_top<0b01, opc, asm, ZPR8, ZPR16>; 4143 def _H : sve2_int_addsub_narrow_high_top<0b10, opc, asm, ZPR16, ZPR32>; 4144 def _S : sve2_int_addsub_narrow_high_top<0b11, opc, asm, ZPR32, ZPR64>; 4145 4146 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _B)>; 4147 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _H)>; 4148 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _S)>; 4149} 4150 4151class sve2_int_sat_extract_narrow_bottom<bits<3> tsz8_64, bits<2> opc, string asm, 4152 ZPRRegOp zprty1, ZPRRegOp zprty2> 4153: I<(outs zprty1:$Zd), (ins zprty2:$Zn), 4154 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4155 bits<5> Zd; 4156 bits<5> Zn; 4157 let Inst{31-23} = 0b010001010; 4158 let Inst{22} = tsz8_64{2}; 4159 let Inst{21} = 0b1; 4160 let Inst{20-19} = tsz8_64{1-0}; 4161 let Inst{18-13} = 0b000010; 4162 let Inst{12-11} = opc; 4163 let Inst{10} = 0b0; 4164 let Inst{9-5} = Zn; 4165 let Inst{4-0} = Zd; 4166} 4167 4168multiclass sve2_int_sat_extract_narrow_bottom<bits<2> opc, string asm, 4169 SDPatternOperator op> { 4170 def _B : sve2_int_sat_extract_narrow_bottom<0b001, opc, asm, ZPR8, ZPR16>; 4171 def _H : sve2_int_sat_extract_narrow_bottom<0b010, opc, asm, ZPR16, ZPR32>; 4172 def _S : sve2_int_sat_extract_narrow_bottom<0b100, opc, asm, ZPR32, ZPR64>; 4173 4174 def : SVE_1_Op_Pat<nxv16i8, op, nxv8i16, !cast<Instruction>(NAME # _B)>; 4175 def : SVE_1_Op_Pat<nxv8i16, op, nxv4i32, !cast<Instruction>(NAME # _H)>; 4176 def : SVE_1_Op_Pat<nxv4i32, op, nxv2i64, !cast<Instruction>(NAME # _S)>; 4177} 4178 4179class sve2_int_sat_extract_narrow_top<bits<3> tsz8_64, bits<2> opc, string asm, 4180 ZPRRegOp zprty1, ZPRRegOp zprty2> 4181: I<(outs zprty1:$Zd), (ins zprty1:$_Zd, zprty2:$Zn), 4182 asm, "\t$Zd, $Zn", "", []>, Sched<[]> { 4183 bits<5> Zd; 4184 bits<5> Zn; 4185 let Inst{31-23} = 0b010001010; 4186 let Inst{22} = tsz8_64{2}; 4187 let Inst{21} = 0b1; 4188 let Inst{20-19} = tsz8_64{1-0}; 4189 let Inst{18-13} = 0b000010; 4190 let Inst{12-11} = opc; 4191 let Inst{10} = 0b1; 4192 let Inst{9-5} = Zn; 4193 let Inst{4-0} = Zd; 4194 4195 let Constraints = "$Zd = $_Zd"; 4196} 4197 4198multiclass sve2_int_sat_extract_narrow_top<bits<2> opc, string asm, 4199 SDPatternOperator op> { 4200 def _B : sve2_int_sat_extract_narrow_top<0b001, opc, asm, ZPR8, ZPR16>; 4201 def _H : sve2_int_sat_extract_narrow_top<0b010, opc, asm, ZPR16, ZPR32>; 4202 def _S : sve2_int_sat_extract_narrow_top<0b100, opc, asm, ZPR32, ZPR64>; 4203 4204 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv8i16, !cast<Instruction>(NAME # _B)>; 4205 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv4i32, !cast<Instruction>(NAME # _H)>; 4206 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4207} 4208 4209//===----------------------------------------------------------------------===// 4210// SVE Integer Arithmetic - Unary Predicated Group 4211//===----------------------------------------------------------------------===// 4212 4213class sve_int_un_pred_arit<bits<2> sz8_64, bits<4> opc, 4214 string asm, ZPRRegOp zprty> 4215: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 4216 asm, "\t$Zd, $Pg/m, $Zn", 4217 "", 4218 []>, Sched<[]> { 4219 bits<3> Pg; 4220 bits<5> Zd; 4221 bits<5> Zn; 4222 let Inst{31-24} = 0b00000100; 4223 let Inst{23-22} = sz8_64; 4224 let Inst{21-20} = 0b01; 4225 let Inst{19} = opc{0}; 4226 let Inst{18-16} = opc{3-1}; 4227 let Inst{15-13} = 0b101; 4228 let Inst{12-10} = Pg; 4229 let Inst{9-5} = Zn; 4230 let Inst{4-0} = Zd; 4231 4232 let Constraints = "$Zd = $_Zd"; 4233 let DestructiveInstType = DestructiveUnaryPassthru; 4234 let ElementSize = zprty.ElementSize; 4235} 4236 4237multiclass sve_int_un_pred_arit_0<bits<3> opc, string asm, 4238 SDPatternOperator op> { 4239 def _B : sve_int_un_pred_arit<0b00, { opc, 0b0 }, asm, ZPR8>, 4240 SVEPseudo2Instr<NAME # _B, 1>; 4241 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4242 SVEPseudo2Instr<NAME # _H, 1>; 4243 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4244 SVEPseudo2Instr<NAME # _S, 1>; 4245 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4246 SVEPseudo2Instr<NAME # _D, 1>; 4247 4248 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4249 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4250 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4251 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4252 4253 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4254 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4255 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4256 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4257 4258 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4259 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4260 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4261 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4262} 4263 4264multiclass sve_int_un_pred_arit_0_h<bits<3> opc, string asm, 4265 SDPatternOperator op> { 4266 def _H : sve_int_un_pred_arit<0b01, { opc, 0b0 }, asm, ZPR16>, 4267 SVEPseudo2Instr<NAME # _H, 1>; 4268 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4269 SVEPseudo2Instr<NAME # _S, 1>; 4270 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4271 SVEPseudo2Instr<NAME # _D, 1>; 4272 4273 def : SVE_InReg_Extend<nxv8i16, op, nxv8i1, nxv8i8, !cast<Instruction>(NAME # _H)>; 4274 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i8, !cast<Instruction>(NAME # _S)>; 4275 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i8, !cast<Instruction>(NAME # _D)>; 4276 4277 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4278 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4279 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4280 4281 defm : SVE_InReg_Extend_PassthruUndef<nxv8i16, op, nxv8i1, nxv8i8, !cast<Pseudo>(NAME # _UNDEF_H)>; 4282 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i8, !cast<Pseudo>(NAME # _UNDEF_S)>; 4283 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i8, !cast<Pseudo>(NAME # _UNDEF_D)>; 4284} 4285 4286multiclass sve_int_un_pred_arit_0_w<bits<3> opc, string asm, 4287 SDPatternOperator op> { 4288 def _S : sve_int_un_pred_arit<0b10, { opc, 0b0 }, asm, ZPR32>, 4289 SVEPseudo2Instr<NAME # _S, 1>; 4290 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4291 SVEPseudo2Instr<NAME # _D, 1>; 4292 4293 def : SVE_InReg_Extend<nxv4i32, op, nxv4i1, nxv4i16, !cast<Instruction>(NAME # _S)>; 4294 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i16, !cast<Instruction>(NAME # _D)>; 4295 4296 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4297 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4298 4299 defm : SVE_InReg_Extend_PassthruUndef<nxv4i32, op, nxv4i1, nxv4i16, !cast<Pseudo>(NAME # _UNDEF_S)>; 4300 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i16, !cast<Pseudo>(NAME # _UNDEF_D)>; 4301} 4302 4303multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm, 4304 SDPatternOperator op> { 4305 def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>, 4306 SVEPseudo2Instr<NAME # _D, 1>; 4307 4308 def : SVE_InReg_Extend<nxv2i64, op, nxv2i1, nxv2i32, !cast<Instruction>(NAME # _D)>; 4309 4310 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4311 4312 defm : SVE_InReg_Extend_PassthruUndef<nxv2i64, op, nxv2i1, nxv2i32, !cast<Pseudo>(NAME # _UNDEF_D)>; 4313} 4314 4315multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm, 4316 SDPatternOperator op> { 4317 def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>, 4318 SVEPseudo2Instr<NAME # _B, 1>; 4319 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4320 SVEPseudo2Instr<NAME # _H, 1>; 4321 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4322 SVEPseudo2Instr<NAME # _S, 1>; 4323 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4324 SVEPseudo2Instr<NAME # _D, 1>; 4325 4326 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4327 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4328 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4329 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4330 4331 def _UNDEF_B : PredOneOpPassthruPseudo<NAME # _B, ZPR8>; 4332 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4333 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4334 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4335 4336 defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 4337 defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4338 defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4339 defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4340} 4341 4342multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm, SDPatternOperator op> { 4343 def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>, 4344 SVEPseudo2Instr<NAME # _H, 1>; 4345 def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>, 4346 SVEPseudo2Instr<NAME # _S, 1>; 4347 def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>, 4348 SVEPseudo2Instr<NAME # _D, 1>; 4349 4350 def : SVE_1_Op_Passthru_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 4351 def : SVE_1_Op_Passthru_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 4352 def : SVE_1_Op_Passthru_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 4353 def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 4354 def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 4355 def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 4356 4357 def _UNDEF_H : PredOneOpPassthruPseudo<NAME # _H, ZPR16>; 4358 def _UNDEF_S : PredOneOpPassthruPseudo<NAME # _S, ZPR32>; 4359 def _UNDEF_D : PredOneOpPassthruPseudo<NAME # _D, ZPR64>; 4360 4361 defm : SVE_1_Op_PassthruUndef_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4362 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4363 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 4364 defm : SVE_1_Op_PassthruUndef_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4365 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 4366 defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 4367} 4368 4369//===----------------------------------------------------------------------===// 4370// SVE Integer Wide Immediate - Unpredicated Group 4371//===----------------------------------------------------------------------===// 4372class sve_int_dup_imm<bits<2> sz8_64, string asm, 4373 ZPRRegOp zprty, Operand immtype> 4374: I<(outs zprty:$Zd), (ins immtype:$imm), 4375 asm, "\t$Zd, $imm", 4376 "", 4377 []>, Sched<[]> { 4378 bits<5> Zd; 4379 bits<9> imm; 4380 let Inst{31-24} = 0b00100101; 4381 let Inst{23-22} = sz8_64; 4382 let Inst{21-14} = 0b11100011; 4383 let Inst{13} = imm{8}; // sh 4384 let Inst{12-5} = imm{7-0}; // imm8 4385 let Inst{4-0} = Zd; 4386 4387 let isReMaterializable = 1; 4388} 4389 4390multiclass sve_int_dup_imm<string asm> { 4391 def _B : sve_int_dup_imm<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8>; 4392 def _H : sve_int_dup_imm<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16>; 4393 def _S : sve_int_dup_imm<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32>; 4394 def _D : sve_int_dup_imm<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64>; 4395 4396 def : InstAlias<"mov $Zd, $imm", 4397 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, cpy_imm8_opt_lsl_i8:$imm), 1>; 4398 def : InstAlias<"mov $Zd, $imm", 4399 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, cpy_imm8_opt_lsl_i16:$imm), 1>; 4400 def : InstAlias<"mov $Zd, $imm", 4401 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, cpy_imm8_opt_lsl_i32:$imm), 1>; 4402 def : InstAlias<"mov $Zd, $imm", 4403 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, cpy_imm8_opt_lsl_i64:$imm), 1>; 4404 4405 def : InstAlias<"fmov $Zd, #0.0", 4406 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, 0, 0), 1>; 4407 def : InstAlias<"fmov $Zd, #0.0", 4408 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, 0, 0), 1>; 4409 def : InstAlias<"fmov $Zd, #0.0", 4410 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, 0, 0), 1>; 4411} 4412 4413class sve_int_dup_fpimm<bits<2> sz8_64, Operand fpimmtype, 4414 string asm, ZPRRegOp zprty> 4415: I<(outs zprty:$Zd), (ins fpimmtype:$imm8), 4416 asm, "\t$Zd, $imm8", 4417 "", 4418 []>, Sched<[]> { 4419 bits<5> Zd; 4420 bits<8> imm8; 4421 let Inst{31-24} = 0b00100101; 4422 let Inst{23-22} = sz8_64; 4423 let Inst{21-14} = 0b11100111; 4424 let Inst{13} = 0b0; 4425 let Inst{12-5} = imm8; 4426 let Inst{4-0} = Zd; 4427 4428 let isReMaterializable = 1; 4429} 4430 4431multiclass sve_int_dup_fpimm<string asm> { 4432 def _H : sve_int_dup_fpimm<0b01, fpimm16, asm, ZPR16>; 4433 def _S : sve_int_dup_fpimm<0b10, fpimm32, asm, ZPR32>; 4434 def _D : sve_int_dup_fpimm<0b11, fpimm64, asm, ZPR64>; 4435 4436 def : InstAlias<"fmov $Zd, $imm8", 4437 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, fpimm16:$imm8), 1>; 4438 def : InstAlias<"fmov $Zd, $imm8", 4439 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, fpimm32:$imm8), 1>; 4440 def : InstAlias<"fmov $Zd, $imm8", 4441 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, fpimm64:$imm8), 1>; 4442} 4443 4444class sve_int_arith_imm0<bits<2> sz8_64, bits<3> opc, string asm, 4445 ZPRRegOp zprty, Operand immtype> 4446: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4447 asm, "\t$Zdn, $_Zdn, $imm", 4448 "", 4449 []>, Sched<[]> { 4450 bits<5> Zdn; 4451 bits<9> imm; 4452 let Inst{31-24} = 0b00100101; 4453 let Inst{23-22} = sz8_64; 4454 let Inst{21-19} = 0b100; 4455 let Inst{18-16} = opc; 4456 let Inst{15-14} = 0b11; 4457 let Inst{13} = imm{8}; // sh 4458 let Inst{12-5} = imm{7-0}; // imm8 4459 let Inst{4-0} = Zdn; 4460 4461 let Constraints = "$Zdn = $_Zdn"; 4462 let DestructiveInstType = DestructiveOther; 4463 let ElementSize = ElementSizeNone; 4464} 4465 4466multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> { 4467 def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8, addsub_imm8_opt_lsl_i8>; 4468 def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>; 4469 def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>; 4470 def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>; 4471 4472 def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8, i32, SVEAddSubImm8Pat, !cast<Instruction>(NAME # _B)>; 4473 def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubImm16Pat, !cast<Instruction>(NAME # _H)>; 4474 def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubImm32Pat, !cast<Instruction>(NAME # _S)>; 4475 def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>; 4476} 4477 4478class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm, 4479 ZPRRegOp zprty, Operand immtype> 4480: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm), 4481 asm, "\t$Zdn, $_Zdn, $imm", 4482 "", 4483 []>, Sched<[]> { 4484 bits<5> Zdn; 4485 bits<8> imm; 4486 let Inst{31-24} = 0b00100101; 4487 let Inst{23-22} = sz8_64; 4488 let Inst{21-16} = opc; 4489 let Inst{15-13} = 0b110; 4490 let Inst{12-5} = imm; 4491 let Inst{4-0} = Zdn; 4492 4493 let Constraints = "$Zdn = $_Zdn"; 4494 let DestructiveInstType = DestructiveOther; 4495 let ElementSize = ElementSizeNone; 4496} 4497 4498multiclass sve_int_arith_imm1<bits<2> opc, string asm, SDPatternOperator op> { 4499 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, simm8_32b>; 4500 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, simm8_32b>; 4501 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, simm8_32b>; 4502 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, simm8_32b>; 4503 4504 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4505 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4506 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4507 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4508} 4509 4510multiclass sve_int_arith_imm1_unsigned<bits<2> opc, string asm, SDPatternOperator op> { 4511 def _B : sve_int_arith_imm<0b00, { 0b1010, opc }, asm, ZPR8, imm0_255>; 4512 def _H : sve_int_arith_imm<0b01, { 0b1010, opc }, asm, ZPR16, imm0_255>; 4513 def _S : sve_int_arith_imm<0b10, { 0b1010, opc }, asm, ZPR32, imm0_255>; 4514 def _D : sve_int_arith_imm<0b11, { 0b1010, opc }, asm, ZPR64, imm0_255>; 4515 4516 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithUImm8Pat, !cast<Instruction>(NAME # _B)>; 4517 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithUImm16Pat, !cast<Instruction>(NAME # _H)>; 4518 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithUImm32Pat, !cast<Instruction>(NAME # _S)>; 4519 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithUImm64Pat, !cast<Instruction>(NAME # _D)>; 4520} 4521 4522multiclass sve_int_arith_imm2<string asm, SDPatternOperator op> { 4523 def _B : sve_int_arith_imm<0b00, 0b110000, asm, ZPR8, simm8_32b>; 4524 def _H : sve_int_arith_imm<0b01, 0b110000, asm, ZPR16, simm8_32b>; 4525 def _S : sve_int_arith_imm<0b10, 0b110000, asm, ZPR32, simm8_32b>; 4526 def _D : sve_int_arith_imm<0b11, 0b110000, asm, ZPR64, simm8_32b>; 4527 4528 def : SVE_1_Op_Imm_Arith_All_Active<nxv16i8, nxv16i1, op, ZPR8, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _B)>; 4529 def : SVE_1_Op_Imm_Arith_All_Active<nxv8i16, nxv8i1, op, ZPR16, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _H)>; 4530 def : SVE_1_Op_Imm_Arith_All_Active<nxv4i32, nxv4i1, op, ZPR32, i32, SVEArithSImmPat32, !cast<Instruction>(NAME # _S)>; 4531 def : SVE_1_Op_Imm_Arith_All_Active<nxv2i64, nxv2i1, op, ZPR64, i64, SVEArithSImmPat64, !cast<Instruction>(NAME # _D)>; 4532} 4533 4534//===----------------------------------------------------------------------===// 4535// SVE Bitwise Logical - Unpredicated Group 4536//===----------------------------------------------------------------------===// 4537 4538class sve_int_bin_cons_log<bits<2> opc, string asm> 4539: I<(outs ZPR64:$Zd), (ins ZPR64:$Zn, ZPR64:$Zm), 4540 asm, "\t$Zd, $Zn, $Zm", 4541 "", 4542 []>, Sched<[]> { 4543 bits<5> Zd; 4544 bits<5> Zm; 4545 bits<5> Zn; 4546 let Inst{31-24} = 0b00000100; 4547 let Inst{23-22} = opc{1-0}; 4548 let Inst{21} = 0b1; 4549 let Inst{20-16} = Zm; 4550 let Inst{15-10} = 0b001100; 4551 let Inst{9-5} = Zn; 4552 let Inst{4-0} = Zd; 4553} 4554 4555multiclass sve_int_bin_cons_log<bits<2> opc, string asm, SDPatternOperator op> { 4556 def NAME : sve_int_bin_cons_log<opc, asm>; 4557 4558 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4559 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4560 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4561 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4562 4563 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4564 (!cast<Instruction>(NAME) ZPR8:$Zd, ZPR8:$Zn, ZPR8:$Zm), 1>; 4565 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4566 (!cast<Instruction>(NAME) ZPR16:$Zd, ZPR16:$Zn, ZPR16:$Zm), 1>; 4567 def : InstAlias<asm # "\t$Zd, $Zn, $Zm", 4568 (!cast<Instruction>(NAME) ZPR32:$Zd, ZPR32:$Zn, ZPR32:$Zm), 1>; 4569} 4570 4571class sve2_int_bitwise_ternary_op_d<bits<3> opc, string asm> 4572: I<(outs ZPR64:$Zdn), (ins ZPR64:$_Zdn, ZPR64:$Zm, ZPR64:$Zk), 4573 asm, "\t$Zdn, $_Zdn, $Zm, $Zk", 4574 "", 4575 []>, Sched<[]> { 4576 bits<5> Zdn; 4577 bits<5> Zk; 4578 bits<5> Zm; 4579 let Inst{31-24} = 0b00000100; 4580 let Inst{23-22} = opc{2-1}; 4581 let Inst{21} = 0b1; 4582 let Inst{20-16} = Zm; 4583 let Inst{15-11} = 0b00111; 4584 let Inst{10} = opc{0}; 4585 let Inst{9-5} = Zk; 4586 let Inst{4-0} = Zdn; 4587 4588 let Constraints = "$Zdn = $_Zdn"; 4589 let DestructiveInstType = DestructiveOther; 4590 let ElementSize = ElementSizeNone; 4591} 4592 4593multiclass sve2_int_bitwise_ternary_op<bits<3> opc, string asm, SDPatternOperator op, 4594 SDPatternOperator ir_op = null_frag> { 4595 def NAME : sve2_int_bitwise_ternary_op_d<opc, asm>; 4596 4597 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4598 (!cast<Instruction>(NAME) ZPR8:$Zdn, ZPR8:$Zm, ZPR8:$Zk), 1>; 4599 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4600 (!cast<Instruction>(NAME) ZPR16:$Zdn, ZPR16:$Zm, ZPR16:$Zk), 1>; 4601 def : InstAlias<asm # "\t$Zdn, $Zdn, $Zm, $Zk", 4602 (!cast<Instruction>(NAME) ZPR32:$Zdn, ZPR32:$Zm, ZPR32:$Zk), 1>; 4603 4604 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4605 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4606 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4607 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4608 4609 4610 def : SVE_3_Op_BSP_Pat<nxv16i8, ir_op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 4611 def : SVE_3_Op_BSP_Pat<nxv8i16, ir_op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 4612 def : SVE_3_Op_BSP_Pat<nxv4i32, ir_op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 4613 def : SVE_3_Op_BSP_Pat<nxv2i64, ir_op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 4614} 4615 4616class sve2_int_rotate_right_imm<bits<4> tsz8_64, string asm, 4617 ZPRRegOp zprty, Operand immtype> 4618: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm, immtype:$imm), 4619 asm, "\t$Zdn, $_Zdn, $Zm, $imm", 4620 "", 4621 []>, Sched<[]> { 4622 bits<5> Zdn; 4623 bits<5> Zm; 4624 bits<6> imm; 4625 let Inst{31-24} = 0b00000100; 4626 let Inst{23-22} = tsz8_64{3-2}; 4627 let Inst{21} = 0b1; 4628 let Inst{20-19} = tsz8_64{1-0}; 4629 let Inst{18-16} = imm{2-0}; // imm3 4630 let Inst{15-10} = 0b001101; 4631 let Inst{9-5} = Zm; 4632 let Inst{4-0} = Zdn; 4633 4634 let Constraints = "$Zdn = $_Zdn"; 4635 let DestructiveInstType = DestructiveOther; 4636 let ElementSize = ElementSizeNone; 4637} 4638 4639multiclass sve2_int_rotate_right_imm<string asm, SDPatternOperator op> { 4640 def _B : sve2_int_rotate_right_imm<{0,0,0,1}, asm, ZPR8, vecshiftR8>; 4641 def _H : sve2_int_rotate_right_imm<{0,0,1,?}, asm, ZPR16, vecshiftR16> { 4642 let Inst{19} = imm{3}; 4643 } 4644 def _S : sve2_int_rotate_right_imm<{0,1,?,?}, asm, ZPR32, vecshiftR32> { 4645 let Inst{20-19} = imm{4-3}; 4646 } 4647 def _D : sve2_int_rotate_right_imm<{1,?,?,?}, asm, ZPR64, vecshiftR64> { 4648 let Inst{22} = imm{5}; 4649 let Inst{20-19} = imm{4-3}; 4650 } 4651 4652 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 4653 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i16, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 4654 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 4655 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i64, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 4656} 4657 4658//===----------------------------------------------------------------------===// 4659// SVE Integer Wide Immediate - Predicated Group 4660//===----------------------------------------------------------------------===// 4661 4662class sve_int_dup_fpimm_pred<bits<2> sz, Operand fpimmtype, 4663 string asm, ZPRRegOp zprty> 4664: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPRAny:$Pg, fpimmtype:$imm8), 4665 asm, "\t$Zd, $Pg/m, $imm8", 4666 "", 4667 []>, Sched<[]> { 4668 bits<4> Pg; 4669 bits<5> Zd; 4670 bits<8> imm8; 4671 let Inst{31-24} = 0b00000101; 4672 let Inst{23-22} = sz; 4673 let Inst{21-20} = 0b01; 4674 let Inst{19-16} = Pg; 4675 let Inst{15-13} = 0b110; 4676 let Inst{12-5} = imm8; 4677 let Inst{4-0} = Zd; 4678 4679 let Constraints = "$Zd = $_Zd"; 4680 let DestructiveInstType = DestructiveOther; 4681 let ElementSize = zprty.ElementSize; 4682} 4683 4684multiclass sve_int_dup_fpimm_pred<string asm> { 4685 def _H : sve_int_dup_fpimm_pred<0b01, fpimm16, asm, ZPR16>; 4686 def _S : sve_int_dup_fpimm_pred<0b10, fpimm32, asm, ZPR32>; 4687 def _D : sve_int_dup_fpimm_pred<0b11, fpimm64, asm, ZPR64>; 4688 4689 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4690 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, fpimm16:$imm8), 1>; 4691 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4692 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, fpimm32:$imm8), 1>; 4693 def : InstAlias<"fmov $Zd, $Pg/m, $imm8", 4694 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, fpimm64:$imm8), 1>; 4695} 4696 4697class sve_int_dup_imm_pred<bits<2> sz8_64, bit m, string asm, 4698 ZPRRegOp zprty, string pred_qual, dag iops> 4699: I<(outs zprty:$Zd), iops, 4700 asm, "\t$Zd, $Pg"#pred_qual#", $imm", 4701 "", []>, Sched<[]> { 4702 bits<5> Zd; 4703 bits<4> Pg; 4704 bits<9> imm; 4705 let Inst{31-24} = 0b00000101; 4706 let Inst{23-22} = sz8_64; 4707 let Inst{21-20} = 0b01; 4708 let Inst{19-16} = Pg; 4709 let Inst{15} = 0b0; 4710 let Inst{14} = m; 4711 let Inst{13} = imm{8}; // sh 4712 let Inst{12-5} = imm{7-0}; // imm8 4713 let Inst{4-0} = Zd; 4714 4715 let DestructiveInstType = DestructiveOther; 4716 let ElementSize = zprty.ElementSize; 4717} 4718 4719multiclass sve_int_dup_imm_pred_merge_inst< 4720 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4721 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4722 let Constraints = "$Zd = $_Zd" in 4723 def NAME : sve_int_dup_imm_pred<sz8_64, 1, asm, zprty, "/m", 4724 (ins zprty:$_Zd, PPRAny:$Pg, cpyimm:$imm)>; 4725 def : InstAlias<"mov $Zd, $Pg/m, $imm", 4726 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4727 def : Pat<(vselect predty:$Pg, 4728 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4729 ZPR:$Zd), 4730 (!cast<Instruction>(NAME) $Zd, $Pg, $imm, $shift)>; 4731} 4732 4733multiclass sve_int_dup_imm_pred_merge<string asm> { 4734 defm _B : sve_int_dup_imm_pred_merge_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4735 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4736 defm _H : sve_int_dup_imm_pred_merge_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4737 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4738 defm _S : sve_int_dup_imm_pred_merge_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4739 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4740 defm _D : sve_int_dup_imm_pred_merge_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4741 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4742 4743 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4744 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPRAny:$Pg, 0, 0), 0>; 4745 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4746 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, 0, 0), 0>; 4747 def : InstAlias<"fmov $Zd, $Pg/m, #0.0", 4748 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, 0, 0), 0>; 4749 4750 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv8f16 ZPR:$Zd)), 4751 (!cast<Instruction>(NAME # _H) $Zd, $Pg, 0, 0)>; 4752 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f16 ZPR:$Zd)), 4753 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4754 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f16 ZPR:$Zd)), 4755 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4756 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv4f32 ZPR:$Zd)), 4757 (!cast<Instruction>(NAME # _S) $Zd, $Pg, 0, 0)>; 4758 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f32 ZPR:$Zd)), 4759 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4760 def : Pat<(vselect PPRAny:$Pg, (SVEDup0), (nxv2f64 ZPR:$Zd)), 4761 (!cast<Instruction>(NAME # _D) $Zd, $Pg, 0, 0)>; 4762} 4763 4764multiclass sve_int_dup_imm_pred_zero_inst< 4765 bits<2> sz8_64, string asm, ZPRRegOp zprty, imm8_opt_lsl cpyimm, 4766 ValueType intty, ValueType predty, ValueType scalarty, ComplexPattern cpx> { 4767 def NAME : sve_int_dup_imm_pred<sz8_64, 0, asm, zprty, "/z", 4768 (ins PPRAny:$Pg, cpyimm:$imm)>; 4769 def : InstAlias<"mov $Zd, $Pg/z, $imm", 4770 (!cast<Instruction>(NAME) zprty:$Zd, PPRAny:$Pg, cpyimm:$imm), 1>; 4771 def : Pat<(intty (zext (predty PPRAny:$Ps1))), 4772 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4773 def : Pat<(intty (sext (predty PPRAny:$Ps1))), 4774 (!cast<Instruction>(NAME) PPRAny:$Ps1, -1, 0)>; 4775 def : Pat<(intty (anyext (predty PPRAny:$Ps1))), 4776 (!cast<Instruction>(NAME) PPRAny:$Ps1, 1, 0)>; 4777 def : Pat<(vselect predty:$Pg, 4778 (intty (splat_vector (scalarty (cpx i32:$imm, i32:$shift)))), 4779 (intty (splat_vector (scalarty 0)))), 4780 (!cast<Instruction>(NAME) $Pg, $imm, $shift)>; 4781} 4782 4783multiclass sve_int_dup_imm_pred_zero<string asm> { 4784 defm _B : sve_int_dup_imm_pred_zero_inst<0b00, asm, ZPR8, cpy_imm8_opt_lsl_i8, 4785 nxv16i8, nxv16i1, i32, SVECpyDupImm8Pat>; 4786 defm _H : sve_int_dup_imm_pred_zero_inst<0b01, asm, ZPR16, cpy_imm8_opt_lsl_i16, 4787 nxv8i16, nxv8i1, i32, SVECpyDupImm16Pat>; 4788 defm _S : sve_int_dup_imm_pred_zero_inst<0b10, asm, ZPR32, cpy_imm8_opt_lsl_i32, 4789 nxv4i32, nxv4i1, i32, SVECpyDupImm32Pat>; 4790 defm _D : sve_int_dup_imm_pred_zero_inst<0b11, asm, ZPR64, cpy_imm8_opt_lsl_i64, 4791 nxv2i64, nxv2i1, i64, SVECpyDupImm64Pat>; 4792} 4793 4794//===----------------------------------------------------------------------===// 4795// SVE Integer Compare - Vectors Group 4796//===----------------------------------------------------------------------===// 4797 4798class sve_int_cmp<bit cmp_1, bits<2> sz8_64, bits<3> opc, string asm, 4799 PPRRegOp pprty, ZPRRegOp zprty1, ZPRRegOp zprty2> 4800: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty1:$Zn, zprty2:$Zm), 4801 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 4802 "", 4803 []>, Sched<[]> { 4804 bits<4> Pd; 4805 bits<3> Pg; 4806 bits<5> Zm; 4807 bits<5> Zn; 4808 let Inst{31-24} = 0b00100100; 4809 let Inst{23-22} = sz8_64; 4810 let Inst{21} = 0b0; 4811 let Inst{20-16} = Zm; 4812 let Inst{15} = opc{2}; 4813 let Inst{14} = cmp_1; 4814 let Inst{13} = opc{1}; 4815 let Inst{12-10} = Pg; 4816 let Inst{9-5} = Zn; 4817 let Inst{4} = opc{0}; 4818 let Inst{3-0} = Pd; 4819 4820 let Defs = [NZCV]; 4821 let ElementSize = pprty.ElementSize; 4822 let isPTestLike = 1; 4823} 4824 4825multiclass SVE_SETCC_Pat<CondCode cc, CondCode invcc, ValueType predvt, 4826 ValueType intvt, Instruction cmp> { 4827 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, cc)), 4828 (cmp $Op1, $Op2, $Op3)>; 4829 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, intvt:$Op3, invcc)), 4830 (cmp $Op1, $Op3, $Op2)>; 4831 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, cc))), 4832 (cmp $Pg, $Op2, $Op3)>; 4833 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op2, intvt:$Op3, invcc))), 4834 (cmp $Pg, $Op3, $Op2)>; 4835} 4836 4837multiclass SVE_SETCC_Pat_With_Zero<CondCode cc, CondCode invcc, ValueType predvt, 4838 ValueType intvt, Instruction cmp> { 4839 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, intvt:$Op2, (SVEDup0), cc)), 4840 (cmp $Op1, $Op2)>; 4841 def : Pat<(predvt (AArch64setcc_z predvt:$Op1, (SVEDup0), intvt:$Op2, invcc)), 4842 (cmp $Op1, $Op2)>; 4843 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), intvt:$Op1, (SVEDup0), cc))), 4844 (cmp $Pg, $Op1)>; 4845 def : Pat<(predvt (and predvt:$Pg, (AArch64setcc_z (predvt (AArch64ptrue 31)), (SVEDup0), intvt:$Op1, invcc))), 4846 (cmp $Pg, $Op1)>; 4847} 4848 4849multiclass sve_int_cmp_0<bits<3> opc, string asm, CondCode cc, CondCode invcc> { 4850 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR8>; 4851 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR16>; 4852 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR32>; 4853 def _D : sve_int_cmp<0b0, 0b11, opc, asm, PPR64, ZPR64, ZPR64>; 4854 4855 defm : SVE_SETCC_Pat<cc, invcc, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 4856 defm : SVE_SETCC_Pat<cc, invcc, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 4857 defm : SVE_SETCC_Pat<cc, invcc, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 4858 defm : SVE_SETCC_Pat<cc, invcc, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 4859} 4860 4861multiclass sve_int_cmp_0_wide<bits<3> opc, string asm, SDPatternOperator op> { 4862 def _B : sve_int_cmp<0b0, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4863 def _H : sve_int_cmp<0b0, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4864 def _S : sve_int_cmp<0b0, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4865 4866 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4867 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4868 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4869} 4870 4871multiclass sve_int_cmp_1_wide<bits<3> opc, string asm, SDPatternOperator op> { 4872 def _B : sve_int_cmp<0b1, 0b00, opc, asm, PPR8, ZPR8, ZPR64>; 4873 def _H : sve_int_cmp<0b1, 0b01, opc, asm, PPR16, ZPR16, ZPR64>; 4874 def _S : sve_int_cmp<0b1, 0b10, opc, asm, PPR32, ZPR32, ZPR64>; 4875 4876 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 4877 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 4878 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 4879} 4880 4881 4882//===----------------------------------------------------------------------===// 4883// SVE Integer Compare - Signed Immediate Group 4884//===----------------------------------------------------------------------===// 4885 4886class sve_int_scmp_vi<bits<2> sz8_64, bits<3> opc, string asm, PPRRegOp pprty, 4887 ZPRRegOp zprty, 4888 Operand immtype> 4889: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm5), 4890 asm, "\t$Pd, $Pg/z, $Zn, $imm5", 4891 "", 4892 []>, Sched<[]> { 4893 bits<4> Pd; 4894 bits<3> Pg; 4895 bits<5> Zn; 4896 bits<5> imm5; 4897 let Inst{31-24} = 0b00100101; 4898 let Inst{23-22} = sz8_64; 4899 let Inst{21} = 0b0; 4900 let Inst{20-16} = imm5; 4901 let Inst{15} = opc{2}; 4902 let Inst{14} = 0b0; 4903 let Inst{13} = opc{1}; 4904 let Inst{12-10} = Pg; 4905 let Inst{9-5} = Zn; 4906 let Inst{4} = opc{0}; 4907 let Inst{3-0} = Pd; 4908 4909 let Defs = [NZCV]; 4910 let ElementSize = pprty.ElementSize; 4911 let isPTestLike = 1; 4912} 4913 4914multiclass SVE_SETCC_Imm_Pat<CondCode cc, CondCode commuted_cc, 4915 ValueType predvt, ValueType intvt, 4916 Operand immtype, Instruction cmp> { 4917 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4918 (intvt ZPR:$Zs1), 4919 (intvt (splat_vector (immtype:$imm))), 4920 cc)), 4921 (cmp $Pg, $Zs1, immtype:$imm)>; 4922 def : Pat<(predvt (AArch64setcc_z (predvt PPR_3b:$Pg), 4923 (intvt (splat_vector (immtype:$imm))), 4924 (intvt ZPR:$Zs1), 4925 commuted_cc)), 4926 (cmp $Pg, $Zs1, immtype:$imm)>; 4927 def : Pat<(predvt (and predvt:$Pg, 4928 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4929 (intvt ZPR:$Zs1), 4930 (intvt (splat_vector (immtype:$imm))), 4931 cc))), 4932 (cmp $Pg, $Zs1, immtype:$imm)>; 4933 def : Pat<(predvt (and predvt:$Pg, 4934 (AArch64setcc_z (predvt (AArch64ptrue 31)), 4935 (intvt (splat_vector (immtype:$imm))), 4936 (intvt ZPR:$Zs1), 4937 commuted_cc))), 4938 (cmp $Pg, $Zs1, immtype:$imm)>; 4939} 4940 4941multiclass sve_int_scmp_vi<bits<3> opc, string asm, CondCode cc, CondCode commuted_cc> { 4942 def _B : sve_int_scmp_vi<0b00, opc, asm, PPR8, ZPR8, simm5_32b>; 4943 def _H : sve_int_scmp_vi<0b01, opc, asm, PPR16, ZPR16, simm5_32b>; 4944 def _S : sve_int_scmp_vi<0b10, opc, asm, PPR32, ZPR32, simm5_32b>; 4945 def _D : sve_int_scmp_vi<0b11, opc, asm, PPR64, ZPR64, simm5_64b>; 4946 4947 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, simm5_32b, 4948 !cast<Instruction>(NAME # _B)>; 4949 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, simm5_32b, 4950 !cast<Instruction>(NAME # _H)>; 4951 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, simm5_32b, 4952 !cast<Instruction>(NAME # _S)>; 4953 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, simm5_64b, 4954 !cast<Instruction>(NAME # _D)>; 4955} 4956 4957 4958//===----------------------------------------------------------------------===// 4959// SVE Integer Compare - Unsigned Immediate Group 4960//===----------------------------------------------------------------------===// 4961 4962class sve_int_ucmp_vi<bits<2> sz8_64, bits<2> opc, string asm, PPRRegOp pprty, 4963 ZPRRegOp zprty, Operand immtype> 4964: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, immtype:$imm7), 4965 asm, "\t$Pd, $Pg/z, $Zn, $imm7", 4966 "", 4967 []>, Sched<[]> { 4968 bits<4> Pd; 4969 bits<3> Pg; 4970 bits<5> Zn; 4971 bits<7> imm7; 4972 let Inst{31-24} = 0b00100100; 4973 let Inst{23-22} = sz8_64; 4974 let Inst{21} = 1; 4975 let Inst{20-14} = imm7; 4976 let Inst{13} = opc{1}; 4977 let Inst{12-10} = Pg; 4978 let Inst{9-5} = Zn; 4979 let Inst{4} = opc{0}; 4980 let Inst{3-0} = Pd; 4981 4982 let Defs = [NZCV]; 4983 let ElementSize = pprty.ElementSize; 4984 let isPTestLike = 1; 4985} 4986 4987multiclass sve_int_ucmp_vi<bits<2> opc, string asm, CondCode cc, 4988 CondCode commuted_cc> { 4989 def _B : sve_int_ucmp_vi<0b00, opc, asm, PPR8, ZPR8, imm0_127>; 4990 def _H : sve_int_ucmp_vi<0b01, opc, asm, PPR16, ZPR16, imm0_127>; 4991 def _S : sve_int_ucmp_vi<0b10, opc, asm, PPR32, ZPR32, imm0_127>; 4992 def _D : sve_int_ucmp_vi<0b11, opc, asm, PPR64, ZPR64, imm0_127_64b>; 4993 4994 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv16i1, nxv16i8, imm0_127, 4995 !cast<Instruction>(NAME # _B)>; 4996 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv8i1, nxv8i16, imm0_127, 4997 !cast<Instruction>(NAME # _H)>; 4998 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv4i1, nxv4i32, imm0_127, 4999 !cast<Instruction>(NAME # _S)>; 5000 defm : SVE_SETCC_Imm_Pat<cc, commuted_cc, nxv2i1, nxv2i64, imm0_127_64b, 5001 !cast<Instruction>(NAME # _D)>; 5002} 5003 5004 5005//===----------------------------------------------------------------------===// 5006// SVE Integer Compare - Scalars Group 5007//===----------------------------------------------------------------------===// 5008 5009class sve_int_cterm<bit sz, bit opc, string asm, RegisterClass rt> 5010: I<(outs), (ins rt:$Rn, rt:$Rm), 5011 asm, "\t$Rn, $Rm", 5012 "", 5013 []>, Sched<[]> { 5014 bits<5> Rm; 5015 bits<5> Rn; 5016 let Inst{31-23} = 0b001001011; 5017 let Inst{22} = sz; 5018 let Inst{21} = 0b1; 5019 let Inst{20-16} = Rm; 5020 let Inst{15-10} = 0b001000; 5021 let Inst{9-5} = Rn; 5022 let Inst{4} = opc; 5023 let Inst{3-0} = 0b0000; 5024 5025 let Defs = [NZCV]; 5026} 5027 5028class sve_int_while_rr<bits<2> sz8_64, bits<4> opc, string asm, 5029 RegisterClass gprty, PPRRegOp pprty> 5030: I<(outs pprty:$Pd), (ins gprty:$Rn, gprty:$Rm), 5031 asm, "\t$Pd, $Rn, $Rm", 5032 "", []>, Sched<[]> { 5033 bits<4> Pd; 5034 bits<5> Rm; 5035 bits<5> Rn; 5036 let Inst{31-24} = 0b00100101; 5037 let Inst{23-22} = sz8_64; 5038 let Inst{21} = 0b1; 5039 let Inst{20-16} = Rm; 5040 let Inst{15-13} = 0b000; 5041 let Inst{12-10} = opc{3-1}; 5042 let Inst{9-5} = Rn; 5043 let Inst{4} = opc{0}; 5044 let Inst{3-0} = Pd; 5045 5046 let Defs = [NZCV]; 5047 let ElementSize = pprty.ElementSize; 5048 let isWhile = 1; 5049} 5050 5051multiclass sve_int_while4_rr<bits<3> opc, string asm, SDPatternOperator op> { 5052 def _B : sve_int_while_rr<0b00, { 0, opc }, asm, GPR32, PPR8>; 5053 def _H : sve_int_while_rr<0b01, { 0, opc }, asm, GPR32, PPR16>; 5054 def _S : sve_int_while_rr<0b10, { 0, opc }, asm, GPR32, PPR32>; 5055 def _D : sve_int_while_rr<0b11, { 0, opc }, asm, GPR32, PPR64>; 5056 5057 def : SVE_2_Op_Pat<nxv16i1, op, i32, i32, !cast<Instruction>(NAME # _B)>; 5058 def : SVE_2_Op_Pat<nxv8i1, op, i32, i32, !cast<Instruction>(NAME # _H)>; 5059 def : SVE_2_Op_Pat<nxv4i1, op, i32, i32, !cast<Instruction>(NAME # _S)>; 5060 def : SVE_2_Op_Pat<nxv2i1, op, i32, i32, !cast<Instruction>(NAME # _D)>; 5061} 5062 5063multiclass sve_int_while8_rr<bits<3> opc, string asm, SDPatternOperator op> { 5064 def _B : sve_int_while_rr<0b00, { 1, opc }, asm, GPR64, PPR8>; 5065 def _H : sve_int_while_rr<0b01, { 1, opc }, asm, GPR64, PPR16>; 5066 def _S : sve_int_while_rr<0b10, { 1, opc }, asm, GPR64, PPR32>; 5067 def _D : sve_int_while_rr<0b11, { 1, opc }, asm, GPR64, PPR64>; 5068 5069 def : SVE_2_Op_Pat<nxv16i1, op, i64, i64, !cast<Instruction>(NAME # _B)>; 5070 def : SVE_2_Op_Pat<nxv8i1, op, i64, i64, !cast<Instruction>(NAME # _H)>; 5071 def : SVE_2_Op_Pat<nxv4i1, op, i64, i64, !cast<Instruction>(NAME # _S)>; 5072 def : SVE_2_Op_Pat<nxv2i1, op, i64, i64, !cast<Instruction>(NAME # _D)>; 5073} 5074 5075class sve2_int_while_rr<bits<2> sz8_64, bits<1> rw, string asm, 5076 PPRRegOp pprty> 5077: I<(outs pprty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 5078 asm, "\t$Pd, $Rn, $Rm", 5079 "", []>, Sched<[]> { 5080 bits<4> Pd; 5081 bits<5> Rm; 5082 bits<5> Rn; 5083 let Inst{31-24} = 0b00100101; 5084 let Inst{23-22} = sz8_64; 5085 let Inst{21} = 0b1; 5086 let Inst{20-16} = Rm; 5087 let Inst{15-10} = 0b001100; 5088 let Inst{9-5} = Rn; 5089 let Inst{4} = rw; 5090 let Inst{3-0} = Pd; 5091 5092 let Defs = [NZCV]; 5093 let ElementSize = pprty.ElementSize; 5094 let isWhile = 1; 5095} 5096 5097multiclass sve2_int_while_rr<bits<1> rw, string asm, string op> { 5098 def _B : sve2_int_while_rr<0b00, rw, asm, PPR8>; 5099 def _H : sve2_int_while_rr<0b01, rw, asm, PPR16>; 5100 def _S : sve2_int_while_rr<0b10, rw, asm, PPR32>; 5101 def _D : sve2_int_while_rr<0b11, rw, asm, PPR64>; 5102 5103 def : SVE_2_Op_Pat<nxv16i1, !cast<SDPatternOperator>(op # _b), i64, i64, !cast<Instruction>(NAME # _B)>; 5104 def : SVE_2_Op_Pat<nxv8i1, !cast<SDPatternOperator>(op # _h), i64, i64, !cast<Instruction>(NAME # _H)>; 5105 def : SVE_2_Op_Pat<nxv4i1, !cast<SDPatternOperator>(op # _s), i64, i64, !cast<Instruction>(NAME # _S)>; 5106 def : SVE_2_Op_Pat<nxv2i1, !cast<SDPatternOperator>(op # _d), i64, i64, !cast<Instruction>(NAME # _D)>; 5107} 5108 5109//===----------------------------------------------------------------------===// 5110// SVE Floating Point Fast Reduction Group 5111//===----------------------------------------------------------------------===// 5112 5113class sve_fp_fast_red<bits<2> sz, bits<3> opc, string asm, 5114 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5115: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 5116 asm, "\t$Vd, $Pg, $Zn", 5117 "", 5118 []>, Sched<[]> { 5119 bits<5> Zn; 5120 bits<5> Vd; 5121 bits<3> Pg; 5122 let Inst{31-24} = 0b01100101; 5123 let Inst{23-22} = sz; 5124 let Inst{21-19} = 0b000; 5125 let Inst{18-16} = opc; 5126 let Inst{15-13} = 0b001; 5127 let Inst{12-10} = Pg; 5128 let Inst{9-5} = Zn; 5129 let Inst{4-0} = Vd; 5130} 5131 5132multiclass sve_fp_fast_red<bits<3> opc, string asm, SDPatternOperator op> { 5133 def _H : sve_fp_fast_red<0b01, opc, asm, ZPR16, FPR16asZPR>; 5134 def _S : sve_fp_fast_red<0b10, opc, asm, ZPR32, FPR32asZPR>; 5135 def _D : sve_fp_fast_red<0b11, opc, asm, ZPR64, FPR64asZPR>; 5136 5137 def : SVE_2_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5138 def : SVE_2_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5139 def : SVE_2_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5140 def : SVE_2_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5141 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5142 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5143} 5144 5145//===----------------------------------------------------------------------===// 5146// SVE Floating Point Accumulating Reduction Group 5147//===----------------------------------------------------------------------===// 5148 5149class sve_fp_2op_p_vd<bits<2> sz, bits<3> opc, string asm, 5150 ZPRRegOp zprty, FPRasZPROperand dstOpType> 5151: I<(outs dstOpType:$Vdn), (ins PPR3bAny:$Pg, dstOpType:$_Vdn, zprty:$Zm), 5152 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 5153 "", 5154 []>, 5155 Sched<[]> { 5156 bits<3> Pg; 5157 bits<5> Vdn; 5158 bits<5> Zm; 5159 let Inst{31-24} = 0b01100101; 5160 let Inst{23-22} = sz; 5161 let Inst{21-19} = 0b011; 5162 let Inst{18-16} = opc; 5163 let Inst{15-13} = 0b001; 5164 let Inst{12-10} = Pg; 5165 let Inst{9-5} = Zm; 5166 let Inst{4-0} = Vdn; 5167 5168 let Constraints = "$Vdn = $_Vdn"; 5169} 5170 5171multiclass sve_fp_2op_p_vd<bits<3> opc, string asm, SDPatternOperator op> { 5172 def _H : sve_fp_2op_p_vd<0b01, opc, asm, ZPR16, FPR16asZPR>; 5173 def _S : sve_fp_2op_p_vd<0b10, opc, asm, ZPR32, FPR32asZPR>; 5174 def _D : sve_fp_2op_p_vd<0b11, opc, asm, ZPR64, FPR64asZPR>; 5175 5176 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Instruction>(NAME # _H)>; 5177 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Instruction>(NAME # _H)>; 5178 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5179 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Instruction>(NAME # _S)>; 5180 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5181 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5182} 5183 5184//===----------------------------------------------------------------------===// 5185// SVE Floating Point Compare - Vectors Group 5186//===----------------------------------------------------------------------===// 5187 5188class sve_fp_3op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5189 ZPRRegOp zprty> 5190: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 5191 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 5192 "", 5193 []>, Sched<[]> { 5194 bits<4> Pd; 5195 bits<3> Pg; 5196 bits<5> Zm; 5197 bits<5> Zn; 5198 let Inst{31-24} = 0b01100101; 5199 let Inst{23-22} = sz; 5200 let Inst{21} = 0b0; 5201 let Inst{20-16} = Zm; 5202 let Inst{15} = opc{2}; 5203 let Inst{14} = 0b1; 5204 let Inst{13} = opc{1}; 5205 let Inst{12-10} = Pg; 5206 let Inst{9-5} = Zn; 5207 let Inst{4} = opc{0}; 5208 let Inst{3-0} = Pd; 5209} 5210 5211multiclass sve_fp_3op_p_pd<bits<3> opc, string asm, SDPatternOperator op> { 5212 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5213 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5214 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5215 5216 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 5217 def : SVE_3_Op_Pat<nxv4i1, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 5218 def : SVE_3_Op_Pat<nxv2i1, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 5219} 5220 5221multiclass sve_fp_3op_p_pd_cc<bits<3> opc, string asm, 5222 CondCode cc1, CondCode cc2, 5223 CondCode invcc1, CondCode invcc2> { 5224 def _H : sve_fp_3op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5225 def _S : sve_fp_3op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5226 def _D : sve_fp_3op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5227 5228 defm : SVE_SETCC_Pat<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5229 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5230 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5231 defm : SVE_SETCC_Pat<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5232 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5233 defm : SVE_SETCC_Pat<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5234 5235 defm : SVE_SETCC_Pat<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5236 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5237 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5238 defm : SVE_SETCC_Pat<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5239 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5240 defm : SVE_SETCC_Pat<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5241} 5242 5243//===----------------------------------------------------------------------===// 5244// SVE Floating Point Compare - with Zero Group 5245//===----------------------------------------------------------------------===// 5246 5247class sve_fp_2op_p_pd<bits<2> sz, bits<3> opc, string asm, PPRRegOp pprty, 5248 ZPRRegOp zprty> 5249: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn), 5250 asm, "\t$Pd, $Pg/z, $Zn, #0.0", 5251 "", 5252 []>, Sched<[]> { 5253 bits<4> Pd; 5254 bits<3> Pg; 5255 bits<5> Zn; 5256 let Inst{31-24} = 0b01100101; 5257 let Inst{23-22} = sz; 5258 let Inst{21-18} = 0b0100; 5259 let Inst{17-16} = opc{2-1}; 5260 let Inst{15-13} = 0b001; 5261 let Inst{12-10} = Pg; 5262 let Inst{9-5} = Zn; 5263 let Inst{4} = opc{0}; 5264 let Inst{3-0} = Pd; 5265} 5266 5267multiclass sve_fp_2op_p_pd<bits<3> opc, string asm, 5268 CondCode cc1, CondCode cc2, 5269 CondCode invcc1, CondCode invcc2> { 5270 def _H : sve_fp_2op_p_pd<0b01, opc, asm, PPR16, ZPR16>; 5271 def _S : sve_fp_2op_p_pd<0b10, opc, asm, PPR32, ZPR32>; 5272 def _D : sve_fp_2op_p_pd<0b11, opc, asm, PPR64, ZPR64>; 5273 5274 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5275 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5276 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5277 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5278 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5279 defm : SVE_SETCC_Pat_With_Zero<cc1, invcc1, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5280 5281 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 5282 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>; 5283 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>; 5284 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 5285 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 5286 defm : SVE_SETCC_Pat_With_Zero<cc2, invcc2, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 5287} 5288 5289 5290//===----------------------------------------------------------------------===// 5291//SVE Index Generation Group 5292//===----------------------------------------------------------------------===// 5293 5294def simm5_8b_tgt : TImmLeaf<i8, [{ return (int8_t)Imm >= -16 && (int8_t)Imm < 16; }]>; 5295def simm5_16b_tgt : TImmLeaf<i16, [{ return (int16_t)Imm >= -16 && (int16_t)Imm < 16; }]>; 5296def simm5_32b_tgt : TImmLeaf<i32, [{ return (int32_t)Imm >= -16 && (int32_t)Imm < 16; }]>; 5297def simm5_64b_tgt : TImmLeaf<i64, [{ return (int64_t)Imm >= -16 && (int64_t)Imm < 16; }]>; 5298def i64imm_32bit_tgt : TImmLeaf<i64, [{ 5299 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); 5300}]>; 5301 5302class sve_int_index_ii<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5303 Operand imm_ty> 5304: I<(outs zprty:$Zd), (ins imm_ty:$imm5, imm_ty:$imm5b), 5305 asm, "\t$Zd, $imm5, $imm5b", 5306 "", []>, Sched<[]> { 5307 bits<5> Zd; 5308 bits<5> imm5; 5309 bits<5> imm5b; 5310 let Inst{31-24} = 0b00000100; 5311 let Inst{23-22} = sz8_64; 5312 let Inst{21} = 0b1; 5313 let Inst{20-16} = imm5b; 5314 let Inst{15-10} = 0b010000; 5315 let Inst{9-5} = imm5; 5316 let Inst{4-0} = Zd; 5317 5318 let isReMaterializable = 1; 5319} 5320 5321multiclass sve_int_index_ii<string asm> { 5322 def _B : sve_int_index_ii<0b00, asm, ZPR8, simm5_8b>; 5323 def _H : sve_int_index_ii<0b01, asm, ZPR16, simm5_16b>; 5324 def _S : sve_int_index_ii<0b10, asm, ZPR32, simm5_32b>; 5325 def _D : sve_int_index_ii<0b11, asm, ZPR64, simm5_64b>; 5326 5327 def : Pat<(nxv16i8 (step_vector simm5_8b_tgt:$imm5b)), 5328 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5329 def : Pat<(nxv8i16 (step_vector simm5_16b_tgt:$imm5b)), 5330 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5331 def : Pat<(nxv4i32 (step_vector simm5_32b_tgt:$imm5b)), 5332 (!cast<Instruction>(NAME # "_S") (i32 0), simm5_32b:$imm5b)>; 5333 def : Pat<(nxv2i64 (step_vector simm5_64b_tgt:$imm5b)), 5334 (!cast<Instruction>(NAME # "_D") (i64 0), simm5_64b:$imm5b)>; 5335 5336 // add(step_vector(step), dup(X)) -> index(X, step). 5337 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5b)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5338 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5339 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5b)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5340 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<SDNodeXForm>("trunc_imm") $imm5b))>; 5341 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5b)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5342 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, simm5_32b:$imm5b)>; 5343 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5b)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5344 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, simm5_64b:$imm5b)>; 5345} 5346 5347class sve_int_index_ir<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5348 RegisterClass srcRegType, Operand imm_ty> 5349: I<(outs zprty:$Zd), (ins imm_ty:$imm5, srcRegType:$Rm), 5350 asm, "\t$Zd, $imm5, $Rm", 5351 "", []>, Sched<[]> { 5352 bits<5> Rm; 5353 bits<5> Zd; 5354 bits<5> imm5; 5355 let Inst{31-24} = 0b00000100; 5356 let Inst{23-22} = sz8_64; 5357 let Inst{21} = 0b1; 5358 let Inst{20-16} = Rm; 5359 let Inst{15-10} = 0b010010; 5360 let Inst{9-5} = imm5; 5361 let Inst{4-0} = Zd; 5362} 5363 5364multiclass sve_int_index_ir<string asm, SDPatternOperator mulop, SDPatternOperator muloneuseop> { 5365 def _B : sve_int_index_ir<0b00, asm, ZPR8, GPR32, simm5_8b>; 5366 def _H : sve_int_index_ir<0b01, asm, ZPR16, GPR32, simm5_16b>; 5367 def _S : sve_int_index_ir<0b10, asm, ZPR32, GPR32, simm5_32b>; 5368 def _D : sve_int_index_ir<0b11, asm, ZPR64, GPR64, simm5_64b>; 5369 5370 def : Pat<(nxv16i8 (step_vector i8:$imm)), 5371 (!cast<Instruction>(NAME # "_B") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5372 def : Pat<(nxv8i16 (step_vector i16:$imm)), 5373 (!cast<Instruction>(NAME # "_H") (i32 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5374 def : Pat<(nxv4i32 (step_vector i32:$imm)), 5375 (!cast<Instruction>(NAME # "_S") (i32 0), (!cast<Instruction>("MOVi32imm") $imm))>; 5376 def : Pat<(nxv2i64 (step_vector i64:$imm)), 5377 (!cast<Instruction>(NAME # "_D") (i64 0), (!cast<Instruction>("MOVi64imm") $imm))>; 5378 def : Pat<(nxv2i64 (step_vector i64imm_32bit_tgt:$imm)), 5379 (!cast<Instruction>(NAME # "_D") (i64 0), (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5380 5381 // add(step_vector(step), dup(X)) -> index(X, step). 5382 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5383 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5384 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5385 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5386 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5387 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, (!cast<Instruction>("MOVi32imm") $imm))>; 5388 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5389 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (!cast<Instruction>("MOVi64imm") $imm))>; 5390 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5391 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5392 5393 // mul(step_vector(1), dup(Y)) -> index(0, Y). 5394 def : Pat<(mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5395 (!cast<Instruction>(NAME # "_B") (i32 0), GPR32:$Rm)>; 5396 def : Pat<(mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5397 (!cast<Instruction>(NAME # "_H") (i32 0), GPR32:$Rm)>; 5398 def : Pat<(mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5399 (!cast<Instruction>(NAME # "_S") (i32 0), GPR32:$Rm)>; 5400 def : Pat<(mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5401 (!cast<Instruction>(NAME # "_D") (i64 0), GPR64:$Rm)>; 5402 5403 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5404 def : Pat<(add (muloneuseop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(simm5_8b:$imm5)))), 5405 (!cast<Instruction>(NAME # "_B") simm5_8b:$imm5, GPR32:$Rm)>; 5406 def : Pat<(add (muloneuseop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), (nxv8i16 (splat_vector(simm5_16b:$imm5)))), 5407 (!cast<Instruction>(NAME # "_H") simm5_16b:$imm5, GPR32:$Rm)>; 5408 def : Pat<(add (muloneuseop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), (nxv4i32 (splat_vector(simm5_32b:$imm5)))), 5409 (!cast<Instruction>(NAME # "_S") simm5_32b:$imm5, GPR32:$Rm)>; 5410 def : Pat<(add (muloneuseop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), (nxv2i64 (splat_vector(simm5_64b:$imm5)))), 5411 (!cast<Instruction>(NAME # "_D") simm5_64b:$imm5, GPR64:$Rm)>; 5412} 5413 5414class sve_int_index_ri<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5415 RegisterClass srcRegType, Operand imm_ty> 5416: I<(outs zprty:$Zd), (ins srcRegType:$Rn, imm_ty:$imm5), 5417 asm, "\t$Zd, $Rn, $imm5", 5418 "", []>, Sched<[]> { 5419 bits<5> Rn; 5420 bits<5> Zd; 5421 bits<5> imm5; 5422 let Inst{31-24} = 0b00000100; 5423 let Inst{23-22} = sz8_64; 5424 let Inst{21} = 0b1; 5425 let Inst{20-16} = imm5; 5426 let Inst{15-10} = 0b010001; 5427 let Inst{9-5} = Rn; 5428 let Inst{4-0} = Zd; 5429} 5430 5431multiclass sve_int_index_ri<string asm> { 5432 def _B : sve_int_index_ri<0b00, asm, ZPR8, GPR32, simm5_8b>; 5433 def _H : sve_int_index_ri<0b01, asm, ZPR16, GPR32, simm5_16b>; 5434 def _S : sve_int_index_ri<0b10, asm, ZPR32, GPR32, simm5_32b>; 5435 def _D : sve_int_index_ri<0b11, asm, ZPR64, GPR64, simm5_64b>; 5436 5437 // add(step_vector(step), dup(X)) -> index(X, step). 5438 def : Pat<(add (nxv16i8 (step_vector_oneuse simm5_8b_tgt:$imm5)), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), 5439 (!cast<Instruction>(NAME # "_B") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5440 def : Pat<(add (nxv8i16 (step_vector_oneuse simm5_16b_tgt:$imm5)), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))), 5441 (!cast<Instruction>(NAME # "_H") GPR32:$Rm, (!cast<SDNodeXForm>("trunc_imm") $imm5))>; 5442 def : Pat<(add (nxv4i32 (step_vector_oneuse simm5_32b_tgt:$imm5)), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))), 5443 (!cast<Instruction>(NAME # "_S") GPR32:$Rm, simm5_32b:$imm5)>; 5444 def : Pat<(add (nxv2i64 (step_vector_oneuse simm5_64b_tgt:$imm5)), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))), 5445 (!cast<Instruction>(NAME # "_D") GPR64:$Rm, simm5_64b:$imm5)>; 5446} 5447 5448class sve_int_index_rr<bits<2> sz8_64, string asm, ZPRRegOp zprty, 5449 RegisterClass srcRegType> 5450: I<(outs zprty:$Zd), (ins srcRegType:$Rn, srcRegType:$Rm), 5451 asm, "\t$Zd, $Rn, $Rm", 5452 "", []>, Sched<[]> { 5453 bits<5> Zd; 5454 bits<5> Rm; 5455 bits<5> Rn; 5456 let Inst{31-24} = 0b00000100; 5457 let Inst{23-22} = sz8_64; 5458 let Inst{21} = 0b1; 5459 let Inst{20-16} = Rm; 5460 let Inst{15-10} = 0b010011; 5461 let Inst{9-5} = Rn; 5462 let Inst{4-0} = Zd; 5463} 5464 5465multiclass sve_int_index_rr<string asm, SDPatternOperator mulop> { 5466 def _B : sve_int_index_rr<0b00, asm, ZPR8, GPR32>; 5467 def _H : sve_int_index_rr<0b01, asm, ZPR16, GPR32>; 5468 def _S : sve_int_index_rr<0b10, asm, ZPR32, GPR32>; 5469 def _D : sve_int_index_rr<0b11, asm, ZPR64, GPR64>; 5470 5471 // add(step_vector(step), dup(X)) -> index(X, step). 5472 def : Pat<(add (nxv16i8 (step_vector_oneuse i8:$imm)), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5473 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5474 def : Pat<(add (nxv8i16 (step_vector_oneuse i16:$imm)), (nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5475 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)))>; 5476 def : Pat<(add (nxv4i32 (step_vector_oneuse i32:$imm)), (nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5477 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, (!cast<Instruction>("MOVi32imm") $imm))>; 5478 def : Pat<(add (nxv2i64 (step_vector_oneuse i64:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5479 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (!cast<Instruction>("MOVi64imm") $imm))>; 5480 def : Pat<(add (nxv2i64 (step_vector_oneuse i64imm_32bit_tgt:$imm)), (nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5481 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, (SUBREG_TO_REG (i64 0), (!cast<Instruction>("MOVi32imm") (!cast<SDNodeXForm>("trunc_imm") $imm)), sub_32))>; 5482 5483 // add(mul(step_vector(1), dup(Y)), dup(X)) -> index(X, Y). 5484 def : Pat<(add (mulop (nxv16i1 (AArch64ptrue 31)), (nxv16i8 (step_vector_oneuse (i8 1))), (nxv16i8 (splat_vector(i32 GPR32:$Rm)))), (nxv16i8 (splat_vector(i32 GPR32:$Rn)))), 5485 (!cast<Instruction>(NAME # "_B") GPR32:$Rn, GPR32:$Rm)>; 5486 def : Pat<(add (mulop (nxv8i1 (AArch64ptrue 31)), (nxv8i16 (step_vector_oneuse (i16 1))), (nxv8i16 (splat_vector(i32 GPR32:$Rm)))),(nxv8i16 (splat_vector(i32 GPR32:$Rn)))), 5487 (!cast<Instruction>(NAME # "_H") GPR32:$Rn, GPR32:$Rm)>; 5488 def : Pat<(add (mulop (nxv4i1 (AArch64ptrue 31)), (nxv4i32 (step_vector_oneuse (i32 1))), (nxv4i32 (splat_vector(i32 GPR32:$Rm)))),(nxv4i32 (splat_vector(i32 GPR32:$Rn)))), 5489 (!cast<Instruction>(NAME # "_S") GPR32:$Rn, GPR32:$Rm)>; 5490 def : Pat<(add (mulop (nxv2i1 (AArch64ptrue 31)), (nxv2i64 (step_vector_oneuse (i64 1))), (nxv2i64 (splat_vector(i64 GPR64:$Rm)))),(nxv2i64 (splat_vector(i64 GPR64:$Rn)))), 5491 (!cast<Instruction>(NAME # "_D") GPR64:$Rn, GPR64:$Rm)>; 5492} 5493 5494//===----------------------------------------------------------------------===// 5495// SVE Bitwise Shift - Predicated Group 5496//===----------------------------------------------------------------------===// 5497 5498class sve_int_bin_pred_shift_imm<bits<4> tsz8_64, bits<4> opc, string asm, 5499 ZPRRegOp zprty, Operand immtype> 5500: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, immtype:$imm), 5501 asm, "\t$Zdn, $Pg/m, $_Zdn, $imm", 5502 "", 5503 []>, Sched<[]> { 5504 bits<3> Pg; 5505 bits<5> Zdn; 5506 bits<6> imm; 5507 let Inst{31-24} = 0b00000100; 5508 let Inst{23-22} = tsz8_64{3-2}; 5509 let Inst{21-20} = 0b00; 5510 let Inst{19-16} = opc; 5511 let Inst{15-13} = 0b100; 5512 let Inst{12-10} = Pg; 5513 let Inst{9-8} = tsz8_64{1-0}; 5514 let Inst{7-5} = imm{2-0}; // imm3 5515 let Inst{4-0} = Zdn; 5516 5517 let Constraints = "$Zdn = $_Zdn"; 5518 let DestructiveInstType = DestructiveBinaryImm; 5519 let ElementSize = zprty.ElementSize; 5520} 5521 5522multiclass sve_int_bin_pred_shift_imm_left<bits<4> opc, string asm, string Ps, 5523 SDPatternOperator op = null_frag> { 5524 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5525 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5526 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5527 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5528 let Inst{8} = imm{3}; 5529 } 5530 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5531 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5532 let Inst{9-8} = imm{4-3}; 5533 } 5534 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5535 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5536 let Inst{22} = imm{5}; 5537 let Inst{9-8} = imm{4-3}; 5538 } 5539 5540 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftL8, !cast<Instruction>(NAME # _B)>; 5541 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftL16, !cast<Instruction>(NAME # _H)>; 5542 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftL32, !cast<Instruction>(NAME # _S)>; 5543 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftL64, !cast<Instruction>(NAME # _D)>; 5544} 5545 5546// As above but shift amount takes the form of a "vector immediate". 5547multiclass sve_int_bin_pred_shift_imm_left_dup<bits<4> opc, string asm, 5548 string Ps, SDPatternOperator op> 5549: sve_int_bin_pred_shift_imm_left<opc, asm, Ps, null_frag> { 5550 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5551 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5552 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5553 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5554} 5555 5556multiclass sve_int_bin_pred_shift_imm_left_zeroing_bhsd<SDPatternOperator op> { 5557 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, tvecshiftL8, FalseLanesZero>; 5558 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, tvecshiftL16, FalseLanesZero>; 5559 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, tvecshiftL32, FalseLanesZero>; 5560 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, tvecshiftL64, FalseLanesZero>; 5561 5562 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftL8, !cast<Pseudo>(NAME # _ZERO_B)>; 5563 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftL16, !cast<Pseudo>(NAME # _ZERO_H)>; 5564 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftL32, !cast<Pseudo>(NAME # _ZERO_S)>; 5565 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftL64, !cast<Pseudo>(NAME # _ZERO_D)>; 5566} 5567 5568multiclass sve_int_bin_pred_shift_imm_right<bits<4> opc, string asm, string Ps, 5569 SDPatternOperator op = null_frag> { 5570 def _B : SVEPseudo2Instr<Ps # _B, 1>, 5571 sve_int_bin_pred_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5572 def _H : SVEPseudo2Instr<Ps # _H, 1>, 5573 sve_int_bin_pred_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5574 let Inst{8} = imm{3}; 5575 } 5576 def _S : SVEPseudo2Instr<Ps # _S, 1>, 5577 sve_int_bin_pred_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5578 let Inst{9-8} = imm{4-3}; 5579 } 5580 def _D : SVEPseudo2Instr<Ps # _D, 1>, 5581 sve_int_bin_pred_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5582 let Inst{22} = imm{5}; 5583 let Inst{9-8} = imm{4-3}; 5584 } 5585 5586 def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i1, nxv16i8, i32, tvecshiftR8, !cast<Instruction>(NAME # _B)>; 5587 def : SVE_3_Op_Imm_Pat<nxv8i16, op, nxv8i1, nxv8i16, i32, tvecshiftR16, !cast<Instruction>(NAME # _H)>; 5588 def : SVE_3_Op_Imm_Pat<nxv4i32, op, nxv4i1, nxv4i32, i32, tvecshiftR32, !cast<Instruction>(NAME # _S)>; 5589 def : SVE_3_Op_Imm_Pat<nxv2i64, op, nxv2i1, nxv2i64, i32, tvecshiftR64, !cast<Instruction>(NAME # _D)>; 5590} 5591 5592// As above but shift amount takes the form of a "vector immediate". 5593multiclass sve_int_bin_pred_shift_imm_right_dup<bits<4> opc, string asm, 5594 string Ps, SDPatternOperator op> 5595: sve_int_bin_pred_shift_imm_right<opc, asm, Ps, null_frag> { 5596 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5597 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5598 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5599 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5600} 5601 5602multiclass sve_int_bin_pred_shift_imm_right_zeroing_bhsd<SDPatternOperator op = null_frag> { 5603 def _ZERO_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, vecshiftR8, FalseLanesZero>; 5604 def _ZERO_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, vecshiftR16, FalseLanesZero>; 5605 def _ZERO_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, vecshiftR32, FalseLanesZero>; 5606 def _ZERO_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, vecshiftR64, FalseLanesZero>; 5607 5608 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv16i8, op, nxv16i1, nxv16i8, tvecshiftR8, !cast<Pseudo>(NAME # _ZERO_B)>; 5609 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv8i16, op, nxv8i1, nxv8i16, tvecshiftR16, !cast<Pseudo>(NAME # _ZERO_H)>; 5610 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv4i32, op, nxv4i1, nxv4i32, tvecshiftR32, !cast<Pseudo>(NAME # _ZERO_S)>; 5611 def : SVE_3_Op_Pat_Shift_Imm_SelZero<nxv2i64, op, nxv2i1, nxv2i64, tvecshiftR64, !cast<Pseudo>(NAME # _ZERO_D)>; 5612} 5613 5614class sve_int_bin_pred_shift<bits<2> sz8_64, bit wide, bits<3> opc, 5615 string asm, ZPRRegOp zprty, ZPRRegOp zprty2> 5616: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty2:$Zm), 5617 asm, "\t$Zdn, $Pg/m, $_Zdn, $Zm", 5618 "", 5619 []>, Sched<[]> { 5620 bits<3> Pg; 5621 bits<5> Zdn; 5622 bits<5> Zm; 5623 let Inst{31-24} = 0b00000100; 5624 let Inst{23-22} = sz8_64; 5625 let Inst{21-20} = 0b01; 5626 let Inst{19} = wide; 5627 let Inst{18-16} = opc; 5628 let Inst{15-13} = 0b100; 5629 let Inst{12-10} = Pg; 5630 let Inst{9-5} = Zm; 5631 let Inst{4-0} = Zdn; 5632 5633 let Constraints = "$Zdn = $_Zdn"; 5634 let DestructiveInstType = DestructiveOther; 5635 let ElementSize = zprty.ElementSize; 5636} 5637 5638multiclass sve_int_bin_pred_shift<bits<3> opc, string asm, string Ps, 5639 SDPatternOperator op, string revname, bit isReverseInstr = 0> { 5640 let DestructiveInstType = DestructiveBinaryCommWithRev in { 5641 def _B : sve_int_bin_pred_shift<0b00, 0b0, opc, asm, ZPR8, ZPR8>, 5642 SVEPseudo2Instr<Ps # _B, 1>, SVEInstr2Rev<NAME # _B, revname # _B, isReverseInstr>; 5643 def _H : sve_int_bin_pred_shift<0b01, 0b0, opc, asm, ZPR16, ZPR16>, 5644 SVEPseudo2Instr<Ps # _H, 1>, SVEInstr2Rev<NAME # _H, revname # _H, isReverseInstr>; 5645 def _S : sve_int_bin_pred_shift<0b10, 0b0, opc, asm, ZPR32, ZPR32>, 5646 SVEPseudo2Instr<Ps # _S, 1>, SVEInstr2Rev<NAME # _S, revname # _S, isReverseInstr>; 5647 def _D : sve_int_bin_pred_shift<0b11, 0b0, opc, asm, ZPR64, ZPR64>, 5648 SVEPseudo2Instr<Ps # _D, 1>, SVEInstr2Rev<NAME # _D, revname # _D, isReverseInstr>; 5649 } 5650 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 5651 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 5652 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 5653 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 5654} 5655 5656multiclass sve_int_bin_pred_zeroing_bhsd<SDPatternOperator op> { 5657 def _ZERO_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesZero>; 5658 def _ZERO_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesZero>; 5659 def _ZERO_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesZero>; 5660 def _ZERO_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesZero>; 5661 5662 def : SVE_3_Op_Pat_SelZero<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _ZERO_B)>; 5663 def : SVE_3_Op_Pat_SelZero<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _ZERO_H)>; 5664 def : SVE_3_Op_Pat_SelZero<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _ZERO_S)>; 5665 def : SVE_3_Op_Pat_SelZero<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _ZERO_D)>; 5666} 5667 5668multiclass sve_int_bin_pred_shift_wide<bits<3> opc, string asm, 5669 SDPatternOperator op> { 5670 def _B : sve_int_bin_pred_shift<0b00, 0b1, opc, asm, ZPR8, ZPR64>; 5671 def _H : sve_int_bin_pred_shift<0b01, 0b1, opc, asm, ZPR16, ZPR64>; 5672 def _S : sve_int_bin_pred_shift<0b10, 0b1, opc, asm, ZPR32, ZPR64>; 5673 5674 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5675 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5676 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5677} 5678 5679//===----------------------------------------------------------------------===// 5680// SVE Shift - Unpredicated Group 5681//===----------------------------------------------------------------------===// 5682 5683class sve_int_bin_cons_shift_wide<bits<2> sz8_64, bits<2> opc, string asm, 5684 ZPRRegOp zprty> 5685: I<(outs zprty:$Zd), (ins zprty:$Zn, ZPR64:$Zm), 5686 asm, "\t$Zd, $Zn, $Zm", 5687 "", 5688 []>, Sched<[]> { 5689 bits<5> Zd; 5690 bits<5> Zm; 5691 bits<5> Zn; 5692 let Inst{31-24} = 0b00000100; 5693 let Inst{23-22} = sz8_64; 5694 let Inst{21} = 0b1; 5695 let Inst{20-16} = Zm; 5696 let Inst{15-12} = 0b1000; 5697 let Inst{11-10} = opc; 5698 let Inst{9-5} = Zn; 5699 let Inst{4-0} = Zd; 5700} 5701 5702multiclass sve_int_bin_cons_shift_wide<bits<2> opc, string asm, SDPatternOperator op> { 5703 def _B : sve_int_bin_cons_shift_wide<0b00, opc, asm, ZPR8>; 5704 def _H : sve_int_bin_cons_shift_wide<0b01, opc, asm, ZPR16>; 5705 def _S : sve_int_bin_cons_shift_wide<0b10, opc, asm, ZPR32>; 5706 5707 def : SVE_2_Op_Pred_All_Active<nxv16i8, op, nxv16i1, nxv16i8, nxv2i64, !cast<Instruction>(NAME # _B)>; 5708 def : SVE_2_Op_Pred_All_Active<nxv8i16, op, nxv8i1, nxv8i16, nxv2i64, !cast<Instruction>(NAME # _H)>; 5709 def : SVE_2_Op_Pred_All_Active<nxv4i32, op, nxv4i1, nxv4i32, nxv2i64, !cast<Instruction>(NAME # _S)>; 5710} 5711 5712class sve_int_bin_cons_shift_imm<bits<4> tsz8_64, bits<2> opc, string asm, 5713 ZPRRegOp zprty, Operand immtype> 5714: I<(outs zprty:$Zd), (ins zprty:$Zn, immtype:$imm), 5715 asm, "\t$Zd, $Zn, $imm", 5716 "", 5717 []>, Sched<[]> { 5718 bits<5> Zd; 5719 bits<5> Zn; 5720 bits<6> imm; 5721 let Inst{31-24} = 0b00000100; 5722 let Inst{23-22} = tsz8_64{3-2}; 5723 let Inst{21} = 0b1; 5724 let Inst{20-19} = tsz8_64{1-0}; 5725 let Inst{18-16} = imm{2-0}; // imm3 5726 let Inst{15-12} = 0b1001; 5727 let Inst{11-10} = opc; 5728 let Inst{9-5} = Zn; 5729 let Inst{4-0} = Zd; 5730} 5731 5732multiclass sve_int_bin_cons_shift_imm_left<bits<2> opc, string asm, 5733 SDPatternOperator op> { 5734 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftL8>; 5735 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftL16> { 5736 let Inst{19} = imm{3}; 5737 } 5738 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftL32> { 5739 let Inst{20-19} = imm{4-3}; 5740 } 5741 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftL64> { 5742 let Inst{22} = imm{5}; 5743 let Inst{20-19} = imm{4-3}; 5744 } 5745 5746 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmL8, !cast<Instruction>(NAME # _B)>; 5747 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmL16, !cast<Instruction>(NAME # _H)>; 5748 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmL32, !cast<Instruction>(NAME # _S)>; 5749 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmL64, !cast<Instruction>(NAME # _D)>; 5750} 5751 5752multiclass sve_int_bin_cons_shift_imm_right<bits<2> opc, string asm, 5753 SDPatternOperator op> { 5754 def _B : sve_int_bin_cons_shift_imm<{0,0,0,1}, opc, asm, ZPR8, vecshiftR8>; 5755 def _H : sve_int_bin_cons_shift_imm<{0,0,1,?}, opc, asm, ZPR16, vecshiftR16> { 5756 let Inst{19} = imm{3}; 5757 } 5758 def _S : sve_int_bin_cons_shift_imm<{0,1,?,?}, opc, asm, ZPR32, vecshiftR32> { 5759 let Inst{20-19} = imm{4-3}; 5760 } 5761 def _D : sve_int_bin_cons_shift_imm<{1,?,?,?}, opc, asm, ZPR64, vecshiftR64> { 5762 let Inst{22} = imm{5}; 5763 let Inst{20-19} = imm{4-3}; 5764 } 5765 5766 def : SVE_Shift_DupImm_All_Active_Pat<nxv16i8, op, nxv16i1, i32, SVEShiftImmR8, !cast<Instruction>(NAME # _B)>; 5767 def : SVE_Shift_DupImm_All_Active_Pat<nxv8i16, op, nxv8i1, i32, SVEShiftImmR16, !cast<Instruction>(NAME # _H)>; 5768 def : SVE_Shift_DupImm_All_Active_Pat<nxv4i32, op, nxv4i1, i32, SVEShiftImmR32, !cast<Instruction>(NAME # _S)>; 5769 def : SVE_Shift_DupImm_All_Active_Pat<nxv2i64, op, nxv2i1, i64, SVEShiftImmR64, !cast<Instruction>(NAME # _D)>; 5770} 5771 5772//===----------------------------------------------------------------------===// 5773// SVE Memory - Store Group 5774//===----------------------------------------------------------------------===// 5775 5776class sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5777 RegisterOperand VecList> 5778: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5779 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5780 "", 5781 []>, Sched<[]> { 5782 bits<3> Pg; 5783 bits<5> Rn; 5784 bits<5> Zt; 5785 bits<4> imm4; 5786 let Inst{31-25} = 0b1110010; 5787 let Inst{24-23} = msz; 5788 let Inst{22-21} = esz; 5789 let Inst{20} = 0; 5790 let Inst{19-16} = imm4; 5791 let Inst{15-13} = 0b111; 5792 let Inst{12-10} = Pg; 5793 let Inst{9-5} = Rn; 5794 let Inst{4-0} = Zt; 5795 5796 let mayStore = 1; 5797} 5798 5799multiclass sve_mem_cst_si<bits<2> msz, bits<2> esz, string asm, 5800 RegisterOperand listty, ZPRRegOp zprty> 5801{ 5802 def NAME : sve_mem_cst_si<msz, esz, asm, listty>; 5803 5804 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5805 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5806 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5807 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5808 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5809 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5810} 5811 5812class sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5813 string asm, Operand immtype> 5814: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5815 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5816 "", 5817 []>, Sched<[]> { 5818 bits<3> Pg; 5819 bits<5> Rn; 5820 bits<5> Zt; 5821 bits<4> imm4; 5822 let Inst{31-25} = 0b1110010; 5823 let Inst{24-23} = sz; 5824 let Inst{22-21} = nregs; 5825 let Inst{20} = 1; 5826 let Inst{19-16} = imm4; 5827 let Inst{15-13} = 0b111; 5828 let Inst{12-10} = Pg; 5829 let Inst{9-5} = Rn; 5830 let Inst{4-0} = Zt; 5831 5832 let mayStore = 1; 5833} 5834 5835multiclass sve_mem_est_si<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5836 string asm, Operand immtype> { 5837 def NAME : sve_mem_est_si<sz, nregs, VecList, asm, immtype>; 5838 5839 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5840 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5841} 5842 5843 5844// SVE store multiple structures (quadwords, scalar plus immediate) 5845class sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList, 5846 string asm, Operand immtype> 5847 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 5848 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5849 "", []>, Sched<[]> { 5850 bits<5> Zt; 5851 bits<5> Rn; 5852 bits<3> Pg; 5853 bits<4> imm4; 5854 let Inst{31-24} = 0b11100100; 5855 let Inst{23-22} = nregs; 5856 let Inst{21-20} = 0b00; 5857 let Inst{19-16} = imm4; 5858 let Inst{15-13} = 0b000; 5859 let Inst{12-10} = Pg; 5860 let Inst{9-5} = Rn; 5861 let Inst{4-0} = Zt; 5862 5863 let mayStore = 1; 5864} 5865 5866multiclass sve_mem_128b_est_si<bits<2> nregs, RegisterOperand VecList, 5867 string asm, Operand immtype> { 5868 def NAME : sve_mem_128b_est_si<nregs, VecList, asm, immtype>; 5869 5870 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5871 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5872} 5873 5874 5875class sve_mem_est_ss<bits<2> sz, bits<2> nregs, RegisterOperand VecList, 5876 string asm, RegisterOperand gprty> 5877: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5878 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5879 "", 5880 []>, Sched<[]> { 5881 bits<3> Pg; 5882 bits<5> Rm; 5883 bits<5> Rn; 5884 bits<5> Zt; 5885 let Inst{31-25} = 0b1110010; 5886 let Inst{24-23} = sz; 5887 let Inst{22-21} = nregs; 5888 let Inst{20-16} = Rm; 5889 let Inst{15-13} = 0b011; 5890 let Inst{12-10} = Pg; 5891 let Inst{9-5} = Rn; 5892 let Inst{4-0} = Zt; 5893 5894 let mayStore = 1; 5895} 5896 5897 5898// SVE store multiple structures (quadwords, scalar plus scalar) 5899class sve_mem_128b_est_ss<bits<2> nregs, RegisterOperand VecList, 5900 string asm, RegisterOperand gprty> 5901 : I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5902 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5903 "", []>, Sched<[]> { 5904 bits<5> Zt; 5905 bits<5> Rn; 5906 bits<3> Pg; 5907 bits<5> Rm; 5908 let Inst{31-24} = 0b11100100; 5909 let Inst{23-22} = nregs; 5910 let Inst{21} = 0b1; 5911 let Inst{20-16} = Rm; 5912 let Inst{15-13} = 0b000; 5913 let Inst{12-10} = Pg; 5914 let Inst{9-5} = Rn; 5915 let Inst{4-0} = Zt; 5916 5917 let mayStore = 1; 5918} 5919 5920 5921class sve_mem_cst_ss_base<bits<4> dtype, string asm, 5922 RegisterOperand listty, RegisterOperand gprty> 5923: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5924 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5925 "", 5926 []>, Sched<[]> { 5927 bits<3> Pg; 5928 bits<5> Rm; 5929 bits<5> Rn; 5930 bits<5> Zt; 5931 let Inst{31-25} = 0b1110010; 5932 let Inst{24-21} = dtype; 5933 let Inst{20-16} = Rm; 5934 let Inst{15-13} = 0b010; 5935 let Inst{12-10} = Pg; 5936 let Inst{9-5} = Rn; 5937 let Inst{4-0} = Zt; 5938 5939 let mayStore = 1; 5940} 5941 5942multiclass sve_mem_cst_ss<bits<4> dtype, string asm, 5943 RegisterOperand listty, ZPRRegOp zprty, 5944 RegisterOperand gprty> { 5945 def NAME : sve_mem_cst_ss_base<dtype, asm, listty, gprty>; 5946 5947 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 5948 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 5949} 5950 5951class sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand VecList> 5952: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 5953 asm, "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5954 "", 5955 []>, Sched<[]> { 5956 bits<3> Pg; 5957 bits<5> Rn; 5958 bits<5> Zt; 5959 bits<4> imm4; 5960 let Inst{31-25} = 0b1110010; 5961 let Inst{24-23} = msz; 5962 let Inst{22-20} = 0b001; 5963 let Inst{19-16} = imm4; 5964 let Inst{15-13} = 0b111; 5965 let Inst{12-10} = Pg; 5966 let Inst{9-5} = Rn; 5967 let Inst{4-0} = Zt; 5968 5969 let mayStore = 1; 5970} 5971 5972multiclass sve_mem_cstnt_si<bits<2> msz, string asm, RegisterOperand listty, 5973 ZPRRegOp zprty> { 5974 def NAME : sve_mem_cstnt_si<msz, asm, listty>; 5975 5976 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5977 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 5978 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $imm4, mul vl]", 5979 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 5980 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn]", 5981 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 5982} 5983 5984class sve_mem_cstnt_ss_base<bits<2> msz, string asm, RegisterOperand listty, 5985 RegisterOperand gprty> 5986: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 5987 asm, "\t$Zt, $Pg, [$Rn, $Rm]", 5988 "", 5989 []>, Sched<[]> { 5990 bits<3> Pg; 5991 bits<5> Rm; 5992 bits<5> Rn; 5993 bits<5> Zt; 5994 let Inst{31-25} = 0b1110010; 5995 let Inst{24-23} = msz; 5996 let Inst{22-21} = 0b00; 5997 let Inst{20-16} = Rm; 5998 let Inst{15-13} = 0b011; 5999 let Inst{12-10} = Pg; 6000 let Inst{9-5} = Rn; 6001 let Inst{4-0} = Zt; 6002 6003 let mayStore = 1; 6004} 6005 6006multiclass sve_mem_cstnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6007 ZPRRegOp zprty, RegisterOperand gprty> { 6008 def NAME : sve_mem_cstnt_ss_base<msz, asm, listty, gprty>; 6009 6010 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Rm]", 6011 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6012} 6013 6014class sve2_mem_sstnt_vs_base<bits<3> opc, string asm, 6015 RegisterOperand listty, ZPRRegOp zprty> 6016: I<(outs), (ins listty:$Zt, PPR3bAny:$Pg, zprty:$Zn, GPR64:$Rm), 6017 asm, "\t$Zt, $Pg, [$Zn, $Rm]", 6018 "", 6019 []>, Sched<[]> { 6020 bits<3> Pg; 6021 bits<5> Rm; 6022 bits<5> Zn; 6023 bits<5> Zt; 6024 let Inst{31-25} = 0b1110010; 6025 let Inst{24-22} = opc; 6026 let Inst{21} = 0b0; 6027 let Inst{20-16} = Rm; 6028 let Inst{15-13} = 0b001; 6029 let Inst{12-10} = Pg; 6030 let Inst{9-5} = Zn; 6031 let Inst{4-0} = Zt; 6032 6033 let mayStore = 1; 6034} 6035 6036multiclass sve2_mem_sstnt_vs_32_ptrs<bits<3> opc, string asm, 6037 SDPatternOperator op, 6038 ValueType vt> { 6039 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_s, ZPR32>; 6040 6041 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 6042 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 6043 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6044 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 6045 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6046 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 6047 6048 def : Pat <(op (nxv4i32 ZPR32:$Zt), (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zn), (i64 GPR64:$Rm), vt), 6049 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm)>; 6050} 6051 6052multiclass sve2_mem_sstnt_vs_64_ptrs<bits<3> opc, string asm, 6053 SDPatternOperator op, 6054 ValueType vt> { 6055 def _REAL : sve2_mem_sstnt_vs_base<opc, asm, Z_d, ZPR64>; 6056 6057 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $Rm]", 6058 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 6059 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6060 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 6061 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6062 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 6063 6064 def : Pat <(op (nxv2i64 ZPR64:$Zt), (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zn), (i64 GPR64:$Rm), vt), 6065 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm)>; 6066} 6067 6068class sve_mem_sst_sv<bits<3> opc, bit xs, bit scaled, string asm, 6069 RegisterOperand VecList, RegisterOperand zprext> 6070: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6071 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 6072 "", 6073 []>, Sched<[]> { 6074 bits<3> Pg; 6075 bits<5> Rn; 6076 bits<5> Zm; 6077 bits<5> Zt; 6078 let Inst{31-25} = 0b1110010; 6079 let Inst{24-22} = opc; 6080 let Inst{21} = scaled; 6081 let Inst{20-16} = Zm; 6082 let Inst{15} = 0b1; 6083 let Inst{14} = xs; 6084 let Inst{13} = 0; 6085 let Inst{12-10} = Pg; 6086 let Inst{9-5} = Rn; 6087 let Inst{4-0} = Zt; 6088 6089 let mayStore = 1; 6090} 6091 6092multiclass sve_mem_32b_sst_sv_32_scaled<bits<3> opc, string asm, 6093 SDPatternOperator sxtw_op, 6094 SDPatternOperator uxtw_op, 6095 RegisterOperand sxtw_opnd, 6096 RegisterOperand uxtw_opnd, 6097 ValueType vt > { 6098 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_s, uxtw_opnd>; 6099 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_s, sxtw_opnd>; 6100 6101 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6102 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6103 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6104 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6105 6106 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6107 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6108 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6109 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6110} 6111 6112multiclass sve_mem_64b_sst_sv_32_scaled<bits<3> opc, string asm, 6113 SDPatternOperator sxtw_op, 6114 SDPatternOperator uxtw_op, 6115 RegisterOperand sxtw_opnd, 6116 RegisterOperand uxtw_opnd, 6117 ValueType vt > { 6118 def _UXTW_SCALED : sve_mem_sst_sv<opc, 0, 1, asm, Z_d, uxtw_opnd>; 6119 def _SXTW_SCALED : sve_mem_sst_sv<opc, 1, 1, asm, Z_d, sxtw_opnd>; 6120 6121 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6122 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6123 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6124 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6125 6126 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6127 (!cast<Instruction>(NAME # _UXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6128 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6129 (!cast<Instruction>(NAME # _SXTW_SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6130} 6131 6132multiclass sve_mem_64b_sst_sv_32_unscaled<bits<3> opc, string asm, 6133 SDPatternOperator sxtw_op, 6134 SDPatternOperator uxtw_op, 6135 RegisterOperand sxtw_opnd, 6136 RegisterOperand uxtw_opnd, 6137 ValueType vt> { 6138 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_d, uxtw_opnd>; 6139 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_d, sxtw_opnd>; 6140 6141 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6142 (!cast<Instruction>(NAME # _UXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6143 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6144 (!cast<Instruction>(NAME # _SXTW) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6145 6146 def : Pat<(uxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6147 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6148 def : Pat<(sxtw_op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6149 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6150} 6151 6152multiclass sve_mem_32b_sst_sv_32_unscaled<bits<3> opc, string asm, 6153 SDPatternOperator sxtw_op, 6154 SDPatternOperator uxtw_op, 6155 RegisterOperand sxtw_opnd, 6156 RegisterOperand uxtw_opnd, 6157 ValueType vt> { 6158 def _UXTW : sve_mem_sst_sv<opc, 0, 0, asm, Z_s, uxtw_opnd>; 6159 def _SXTW : sve_mem_sst_sv<opc, 1, 0, asm, Z_s, sxtw_opnd>; 6160 6161 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6162 (!cast<Instruction>(NAME # _UXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 6163 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6164 (!cast<Instruction>(NAME # _SXTW) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 6165 6166 def : Pat<(uxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6167 (!cast<Instruction>(NAME # _UXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6168 def : Pat<(sxtw_op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt), 6169 (!cast<Instruction>(NAME # _SXTW) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6170} 6171 6172class sve_mem_sst_sv2<bits<2> msz, bit scaled, string asm, 6173 RegisterOperand zprext> 6174: I<(outs), (ins Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 6175 asm, "\t$Zt, $Pg, [$Rn, $Zm]", 6176 "", 6177 []>, Sched<[]> { 6178 bits<3> Pg; 6179 bits<5> Rn; 6180 bits<5> Zm; 6181 bits<5> Zt; 6182 let Inst{31-25} = 0b1110010; 6183 let Inst{24-23} = msz; 6184 let Inst{22} = 0b0; 6185 let Inst{21} = scaled; 6186 let Inst{20-16} = Zm; 6187 let Inst{15-13} = 0b101; 6188 let Inst{12-10} = Pg; 6189 let Inst{9-5} = Rn; 6190 let Inst{4-0} = Zt; 6191 6192 let mayStore = 1; 6193} 6194 6195multiclass sve_mem_sst_sv_64_scaled<bits<2> msz, string asm, 6196 SDPatternOperator op, 6197 RegisterOperand zprext, 6198 ValueType vt> { 6199 def _SCALED : sve_mem_sst_sv2<msz, 1, asm, zprext>; 6200 6201 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6202 (!cast<Instruction>(NAME # _SCALED) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 6203 6204 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt), 6205 (!cast<Instruction>(NAME # _SCALED) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 6206} 6207 6208multiclass sve_mem_sst_sv_64_unscaled<bits<2> msz, string asm, 6209 SDPatternOperator op, 6210 ValueType vt> { 6211 def NAME : sve_mem_sst_sv2<msz, 0, asm, ZPR64ExtLSL8>; 6212 6213 def : InstAlias<asm # "\t$Zt, $Pg, [$Rn, $Zm]", 6214 (!cast<Instruction>(NAME) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 6215 6216 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt), 6217 (!cast<Instruction>(NAME) ZPR:$data, PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 6218} 6219 6220class sve_mem_sst_vi<bits<3> opc, string asm, ZPRRegOp zprty, 6221 RegisterOperand VecList, Operand imm_ty> 6222: I<(outs), (ins VecList:$Zt, PPR3bAny:$Pg, zprty:$Zn, imm_ty:$imm5), 6223 asm, "\t$Zt, $Pg, [$Zn, $imm5]", 6224 "", 6225 []>, Sched<[]> { 6226 bits<3> Pg; 6227 bits<5> imm5; 6228 bits<5> Zn; 6229 bits<5> Zt; 6230 let Inst{31-25} = 0b1110010; 6231 let Inst{24-23} = opc{2-1}; 6232 let Inst{22} = 0b1; 6233 let Inst{21} = opc{0}; 6234 let Inst{20-16} = imm5; 6235 let Inst{15-13} = 0b101; 6236 let Inst{12-10} = Pg; 6237 let Inst{9-5} = Zn; 6238 let Inst{4-0} = Zt; 6239 6240 let mayStore = 1; 6241} 6242 6243multiclass sve_mem_32b_sst_vi_ptrs<bits<3> opc, string asm, 6244 Operand imm_ty, 6245 SDPatternOperator op, 6246 ValueType vt> { 6247 def _IMM : sve_mem_sst_vi<opc, asm, ZPR32, Z_s, imm_ty>; 6248 6249 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6250 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 6251 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6252 (!cast<Instruction>(NAME # _IMM) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 6253 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6254 (!cast<Instruction>(NAME # _IMM) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 6255 6256 def : Pat<(op (nxv4i32 ZPR:$data), (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt), 6257 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6258} 6259 6260multiclass sve_mem_64b_sst_vi_ptrs<bits<3> opc, string asm, 6261 Operand imm_ty, 6262 SDPatternOperator op, 6263 ValueType vt> { 6264 def _IMM : sve_mem_sst_vi<opc, asm, ZPR64, Z_d, imm_ty>; 6265 6266 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6267 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 6268 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn, $imm5]", 6269 (!cast<Instruction>(NAME # _IMM) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 6270 def : InstAlias<asm # "\t$Zt, $Pg, [$Zn]", 6271 (!cast<Instruction>(NAME # _IMM) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 6272 6273 def : Pat<(op (nxv2i64 ZPR:$data), (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt), 6274 (!cast<Instruction>(NAME # _IMM) ZPR:$data, PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 6275} 6276 6277class sve_mem_z_spill<string asm> 6278: I<(outs), (ins ZPRAny:$Zt, GPR64sp:$Rn, simm9:$imm9), 6279 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 6280 "", 6281 []>, Sched<[]> { 6282 bits<5> Rn; 6283 bits<5> Zt; 6284 bits<9> imm9; 6285 let Inst{31-22} = 0b1110010110; 6286 let Inst{21-16} = imm9{8-3}; 6287 let Inst{15-13} = 0b010; 6288 let Inst{12-10} = imm9{2-0}; 6289 let Inst{9-5} = Rn; 6290 let Inst{4-0} = Zt; 6291 6292 let mayStore = 1; 6293} 6294 6295multiclass sve_mem_z_spill<string asm> { 6296 def NAME : sve_mem_z_spill<asm>; 6297 6298 def : InstAlias<asm # "\t$Zt, [$Rn]", 6299 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 6300} 6301 6302class sve_mem_p_spill<string asm> 6303: I<(outs), (ins PPRAny:$Pt, GPR64sp:$Rn, simm9:$imm9), 6304 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 6305 "", 6306 []>, Sched<[]> { 6307 bits<4> Pt; 6308 bits<5> Rn; 6309 bits<9> imm9; 6310 let Inst{31-22} = 0b1110010110; 6311 let Inst{21-16} = imm9{8-3}; 6312 let Inst{15-13} = 0b000; 6313 let Inst{12-10} = imm9{2-0}; 6314 let Inst{9-5} = Rn; 6315 let Inst{4} = 0b0; 6316 let Inst{3-0} = Pt; 6317 6318 let mayStore = 1; 6319} 6320 6321multiclass sve_mem_p_spill<string asm> { 6322 def NAME : sve_mem_p_spill<asm>; 6323 6324 def : InstAlias<asm # "\t$Pt, [$Rn]", 6325 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 6326} 6327 6328//===----------------------------------------------------------------------===// 6329// SVE Permute - Predicates Group 6330//===----------------------------------------------------------------------===// 6331 6332class sve_int_perm_bin_perm_pp<bits<3> opc, bits<2> sz8_64, string asm, 6333 PPRRegOp pprty, SDPatternOperator op> 6334: I<(outs pprty:$Pd), (ins pprty:$Pn, pprty:$Pm), 6335 asm, "\t$Pd, $Pn, $Pm", 6336 "", 6337 [(set nxv16i1:$Pd, (op nxv16i1:$Pn, nxv16i1:$Pm))]>, Sched<[]> { 6338 bits<4> Pd; 6339 bits<4> Pm; 6340 bits<4> Pn; 6341 let Inst{31-24} = 0b00000101; 6342 let Inst{23-22} = sz8_64; 6343 let Inst{21-20} = 0b10; 6344 let Inst{19-16} = Pm; 6345 let Inst{15-13} = 0b010; 6346 let Inst{12-10} = opc; 6347 let Inst{9} = 0b0; 6348 let Inst{8-5} = Pn; 6349 let Inst{4} = 0b0; 6350 let Inst{3-0} = Pd; 6351} 6352 6353multiclass sve_int_perm_bin_perm_pp<bits<3> opc, string asm, 6354 SDPatternOperator ir_op, 6355 SDPatternOperator op_b16, 6356 SDPatternOperator op_b32, 6357 SDPatternOperator op_b64> { 6358 def _B : sve_int_perm_bin_perm_pp<opc, 0b00, asm, PPR8, ir_op>; 6359 def _H : sve_int_perm_bin_perm_pp<opc, 0b01, asm, PPR16, op_b16>; 6360 def _S : sve_int_perm_bin_perm_pp<opc, 0b10, asm, PPR32, op_b32>; 6361 def _D : sve_int_perm_bin_perm_pp<opc, 0b11, asm, PPR64, op_b64>; 6362 6363 def : SVE_2_Op_Pat<nxv8i1, ir_op, nxv8i1, nxv8i1, !cast<Instruction>(NAME # _H)>; 6364 def : SVE_2_Op_Pat<nxv4i1, ir_op, nxv4i1, nxv4i1, !cast<Instruction>(NAME # _S)>; 6365 def : SVE_2_Op_Pat<nxv2i1, ir_op, nxv2i1, nxv2i1, !cast<Instruction>(NAME # _D)>; 6366} 6367 6368class sve_int_perm_punpk<bit opc, string asm> 6369: I<(outs PPR16:$Pd), (ins PPR8:$Pn), 6370 asm, "\t$Pd, $Pn", 6371 "", 6372 []>, Sched<[]> { 6373 bits<4> Pd; 6374 bits<4> Pn; 6375 let Inst{31-17} = 0b000001010011000; 6376 let Inst{16} = opc; 6377 let Inst{15-9} = 0b0100000; 6378 let Inst{8-5} = Pn; 6379 let Inst{4} = 0b0; 6380 let Inst{3-0} = Pd; 6381} 6382 6383multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> { 6384 def NAME : sve_int_perm_punpk<opc, asm>; 6385 6386 def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>; 6387 def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1, !cast<Instruction>(NAME)>; 6388 def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1, !cast<Instruction>(NAME)>; 6389} 6390 6391class sve_int_rdffr_pred<bit s, string asm> 6392: I<(outs PPR8:$Pd), (ins PPRAny:$Pg), 6393 asm, "\t$Pd, $Pg/z", 6394 "", 6395 []>, Sched<[]> { 6396 bits<4> Pd; 6397 bits<4> Pg; 6398 let Inst{31-23} = 0b001001010; 6399 let Inst{22} = s; 6400 let Inst{21-9} = 0b0110001111000; 6401 let Inst{8-5} = Pg; 6402 let Inst{4} = 0; 6403 let Inst{3-0} = Pd; 6404 6405 let Defs = !if(s, [NZCV], []); 6406 let Uses = [FFR]; 6407} 6408 6409multiclass sve_int_rdffr_pred<bit s, string asm, SDPatternOperator op> { 6410 def _REAL : sve_int_rdffr_pred<s, asm>; 6411 6412 // We need a layer of indirection because early machine code passes balk at 6413 // physical register (i.e. FFR) uses that have no previous definition. 6414 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6415 def "" : Pseudo<(outs PPR8:$Pd), (ins PPRAny:$Pg), [(set (nxv16i1 PPR8:$Pd), (op (nxv16i1 PPRAny:$Pg)))]>, 6416 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd, PPRAny:$Pg)>; 6417 } 6418} 6419 6420class sve_int_rdffr_unpred<string asm> : I< 6421 (outs PPR8:$Pd), (ins), 6422 asm, "\t$Pd", 6423 "", 6424 []>, Sched<[]> { 6425 bits<4> Pd; 6426 let Inst{31-4} = 0b0010010100011001111100000000; 6427 let Inst{3-0} = Pd; 6428 6429 let Uses = [FFR]; 6430} 6431 6432multiclass sve_int_rdffr_unpred<string asm, SDPatternOperator op> { 6433 def _REAL : sve_int_rdffr_unpred<asm>; 6434 6435 // We need a layer of indirection because early machine code passes balk at 6436 // physical register (i.e. FFR) uses that have no previous definition. 6437 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 6438 def "" : Pseudo<(outs PPR8:$Pd), (ins), [(set (nxv16i1 PPR8:$Pd), (op))]>, 6439 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) PPR8:$Pd)>; 6440 } 6441} 6442 6443class sve_int_wrffr<string asm, SDPatternOperator op> 6444: I<(outs), (ins PPR8:$Pn), 6445 asm, "\t$Pn", 6446 "", 6447 [(op (nxv16i1 PPR8:$Pn))]>, Sched<[]> { 6448 bits<4> Pn; 6449 let Inst{31-9} = 0b00100101001010001001000; 6450 let Inst{8-5} = Pn; 6451 let Inst{4-0} = 0b00000; 6452 6453 let hasSideEffects = 1; 6454 let Defs = [FFR]; 6455} 6456 6457class sve_int_setffr<string asm, SDPatternOperator op> 6458: I<(outs), (ins), 6459 asm, "", 6460 "", 6461 [(op)]>, Sched<[]> { 6462 let Inst{31-0} = 0b00100101001011001001000000000000; 6463 6464 let hasSideEffects = 1; 6465 let Defs = [FFR]; 6466} 6467 6468//===----------------------------------------------------------------------===// 6469// SVE Permute Vector - Predicated Group 6470//===----------------------------------------------------------------------===// 6471 6472class sve_int_perm_clast_rz<bits<2> sz8_64, bit ab, string asm, 6473 ZPRRegOp zprty, RegisterClass rt> 6474: I<(outs rt:$Rdn), (ins PPR3bAny:$Pg, rt:$_Rdn, zprty:$Zm), 6475 asm, "\t$Rdn, $Pg, $_Rdn, $Zm", 6476 "", 6477 []>, Sched<[]> { 6478 bits<3> Pg; 6479 bits<5> Rdn; 6480 bits<5> Zm; 6481 let Inst{31-24} = 0b00000101; 6482 let Inst{23-22} = sz8_64; 6483 let Inst{21-17} = 0b11000; 6484 let Inst{16} = ab; 6485 let Inst{15-13} = 0b101; 6486 let Inst{12-10} = Pg; 6487 let Inst{9-5} = Zm; 6488 let Inst{4-0} = Rdn; 6489 6490 let Constraints = "$Rdn = $_Rdn"; 6491} 6492 6493multiclass sve_int_perm_clast_rz<bit ab, string asm, SDPatternOperator op> { 6494 def _B : sve_int_perm_clast_rz<0b00, ab, asm, ZPR8, GPR32>; 6495 def _H : sve_int_perm_clast_rz<0b01, ab, asm, ZPR16, GPR32>; 6496 def _S : sve_int_perm_clast_rz<0b10, ab, asm, ZPR32, GPR32>; 6497 def _D : sve_int_perm_clast_rz<0b11, ab, asm, ZPR64, GPR64>; 6498 6499 def : SVE_3_Op_Pat<i32, op, nxv16i1, i32, nxv16i8, !cast<Instruction>(NAME # _B)>; 6500 def : SVE_3_Op_Pat<i32, op, nxv8i1, i32, nxv8i16, !cast<Instruction>(NAME # _H)>; 6501 def : SVE_3_Op_Pat<i32, op, nxv4i1, i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6502 def : SVE_3_Op_Pat<i64, op, nxv2i1, i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6503} 6504 6505class sve_int_perm_clast_vz<bits<2> sz8_64, bit ab, string asm, 6506 ZPRRegOp zprty, RegisterClass rt> 6507: I<(outs rt:$Vdn), (ins PPR3bAny:$Pg, rt:$_Vdn, zprty:$Zm), 6508 asm, "\t$Vdn, $Pg, $_Vdn, $Zm", 6509 "", 6510 []>, Sched<[]> { 6511 bits<3> Pg; 6512 bits<5> Vdn; 6513 bits<5> Zm; 6514 let Inst{31-24} = 0b00000101; 6515 let Inst{23-22} = sz8_64; 6516 let Inst{21-17} = 0b10101; 6517 let Inst{16} = ab; 6518 let Inst{15-13} = 0b100; 6519 let Inst{12-10} = Pg; 6520 let Inst{9-5} = Zm; 6521 let Inst{4-0} = Vdn; 6522 6523 let Constraints = "$Vdn = $_Vdn"; 6524} 6525 6526multiclass sve_int_perm_clast_vz<bit ab, string asm, SDPatternOperator op> { 6527 def _B : sve_int_perm_clast_vz<0b00, ab, asm, ZPR8, FPR8>; 6528 def _H : sve_int_perm_clast_vz<0b01, ab, asm, ZPR16, FPR16>; 6529 def _S : sve_int_perm_clast_vz<0b10, ab, asm, ZPR32, FPR32>; 6530 def _D : sve_int_perm_clast_vz<0b11, ab, asm, ZPR64, FPR64>; 6531 6532 def : SVE_3_Op_Pat<f16, op, nxv8i1, f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6533 def : SVE_3_Op_Pat<f32, op, nxv4i1, f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6534 def : SVE_3_Op_Pat<f64, op, nxv2i1, f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6535 6536 def : SVE_3_Op_Pat<bf16, op, nxv8i1, bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6537} 6538 6539class sve_int_perm_clast_zz<bits<2> sz8_64, bit ab, string asm, 6540 ZPRRegOp zprty> 6541: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6542 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6543 "", 6544 []>, Sched<[]> { 6545 bits<3> Pg; 6546 bits<5> Zdn; 6547 bits<5> Zm; 6548 let Inst{31-24} = 0b00000101; 6549 let Inst{23-22} = sz8_64; 6550 let Inst{21-17} = 0b10100; 6551 let Inst{16} = ab; 6552 let Inst{15-13} = 0b100; 6553 let Inst{12-10} = Pg; 6554 let Inst{9-5} = Zm; 6555 let Inst{4-0} = Zdn; 6556 6557 let Constraints = "$Zdn = $_Zdn"; 6558 let DestructiveInstType = DestructiveOther; 6559 let ElementSize = ElementSizeNone; 6560} 6561 6562multiclass sve_int_perm_clast_zz<bit ab, string asm, SDPatternOperator op> { 6563 def _B : sve_int_perm_clast_zz<0b00, ab, asm, ZPR8>; 6564 def _H : sve_int_perm_clast_zz<0b01, ab, asm, ZPR16>; 6565 def _S : sve_int_perm_clast_zz<0b10, ab, asm, ZPR32>; 6566 def _D : sve_int_perm_clast_zz<0b11, ab, asm, ZPR64>; 6567 6568 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6569 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6570 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6571 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6572 6573 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6574 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6575 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6576 6577 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6578} 6579 6580class sve_int_perm_last_r<bits<2> sz8_64, bit ab, string asm, 6581 ZPRRegOp zprty, RegisterClass resultRegType> 6582: I<(outs resultRegType:$Rd), (ins PPR3bAny:$Pg, zprty:$Zn), 6583 asm, "\t$Rd, $Pg, $Zn", 6584 "", 6585 []>, Sched<[]> { 6586 bits<3> Pg; 6587 bits<5> Rd; 6588 bits<5> Zn; 6589 let Inst{31-24} = 0b00000101; 6590 let Inst{23-22} = sz8_64; 6591 let Inst{21-17} = 0b10000; 6592 let Inst{16} = ab; 6593 let Inst{15-13} = 0b101; 6594 let Inst{12-10} = Pg; 6595 let Inst{9-5} = Zn; 6596 let Inst{4-0} = Rd; 6597} 6598 6599multiclass sve_int_perm_last_r<bit ab, string asm, SDPatternOperator op> { 6600 def _B : sve_int_perm_last_r<0b00, ab, asm, ZPR8, GPR32>; 6601 def _H : sve_int_perm_last_r<0b01, ab, asm, ZPR16, GPR32>; 6602 def _S : sve_int_perm_last_r<0b10, ab, asm, ZPR32, GPR32>; 6603 def _D : sve_int_perm_last_r<0b11, ab, asm, ZPR64, GPR64>; 6604 6605 def : SVE_2_Op_Pat<i32, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6606 def : SVE_2_Op_Pat<i32, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6607 def : SVE_2_Op_Pat<i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6608 def : SVE_2_Op_Pat<i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6609} 6610 6611class sve_int_perm_last_v<bits<2> sz8_64, bit ab, string asm, 6612 ZPRRegOp zprty, RegisterClass dstRegtype> 6613: I<(outs dstRegtype:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 6614 asm, "\t$Vd, $Pg, $Zn", 6615 "", 6616 []>, Sched<[]> { 6617 bits<3> Pg; 6618 bits<5> Vd; 6619 bits<5> Zn; 6620 let Inst{31-24} = 0b00000101; 6621 let Inst{23-22} = sz8_64; 6622 let Inst{21-17} = 0b10001; 6623 let Inst{16} = ab; 6624 let Inst{15-13} = 0b100; 6625 let Inst{12-10} = Pg; 6626 let Inst{9-5} = Zn; 6627 let Inst{4-0} = Vd; 6628} 6629 6630multiclass sve_int_perm_last_v<bit ab, string asm, SDPatternOperator op> { 6631 def _B : sve_int_perm_last_v<0b00, ab, asm, ZPR8, FPR8>; 6632 def _H : sve_int_perm_last_v<0b01, ab, asm, ZPR16, FPR16>; 6633 def _S : sve_int_perm_last_v<0b10, ab, asm, ZPR32, FPR32>; 6634 def _D : sve_int_perm_last_v<0b11, ab, asm, ZPR64, FPR64>; 6635 6636 def : SVE_2_Op_Pat<f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>; 6637 def : SVE_2_Op_Pat<f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6638 def : SVE_2_Op_Pat<f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>; 6639 def : SVE_2_Op_Pat<f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6640 6641 def : SVE_2_Op_Pat<bf16, op, nxv8i1, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6642} 6643 6644class sve_int_perm_splice<bits<2> sz8_64, string asm, ZPRRegOp zprty> 6645: I<(outs zprty:$Zdn), (ins PPR3bAny:$Pg, zprty:$_Zdn, zprty:$Zm), 6646 asm, "\t$Zdn, $Pg, $_Zdn, $Zm", 6647 "", 6648 []>, Sched<[]> { 6649 bits<3> Pg; 6650 bits<5> Zdn; 6651 bits<5> Zm; 6652 let Inst{31-24} = 0b00000101; 6653 let Inst{23-22} = sz8_64; 6654 let Inst{21-13} = 0b101100100; 6655 let Inst{12-10} = Pg; 6656 let Inst{9-5} = Zm; 6657 let Inst{4-0} = Zdn; 6658 6659 let Constraints = "$Zdn = $_Zdn"; 6660 let DestructiveInstType = DestructiveOther; 6661 let ElementSize = ElementSizeNone; 6662} 6663 6664multiclass sve_int_perm_splice<string asm, SDPatternOperator op> { 6665 def _B : sve_int_perm_splice<0b00, asm, ZPR8>; 6666 def _H : sve_int_perm_splice<0b01, asm, ZPR16>; 6667 def _S : sve_int_perm_splice<0b10, asm, ZPR32>; 6668 def _D : sve_int_perm_splice<0b11, asm, ZPR64>; 6669 6670 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 6671 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 6672 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 6673 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 6674 6675 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 6676 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 6677 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 6678 6679 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8i1, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME # _H)>; 6680} 6681 6682class sve2_int_perm_splice_cons<bits<2> sz8_64, string asm, 6683 ZPRRegOp zprty, RegisterOperand VecList> 6684: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, VecList:$Zn), 6685 asm, "\t$Zd, $Pg, $Zn", 6686 "", 6687 []>, Sched<[]> { 6688 bits<3> Pg; 6689 bits<5> Zn; 6690 bits<5> Zd; 6691 let Inst{31-24} = 0b00000101; 6692 let Inst{23-22} = sz8_64; 6693 let Inst{21-13} = 0b101101100; 6694 let Inst{12-10} = Pg; 6695 let Inst{9-5} = Zn; 6696 let Inst{4-0} = Zd; 6697} 6698 6699multiclass sve2_int_perm_splice_cons<string asm> { 6700 def _B : sve2_int_perm_splice_cons<0b00, asm, ZPR8, ZZ_b>; 6701 def _H : sve2_int_perm_splice_cons<0b01, asm, ZPR16, ZZ_h>; 6702 def _S : sve2_int_perm_splice_cons<0b10, asm, ZPR32, ZZ_s>; 6703 def _D : sve2_int_perm_splice_cons<0b11, asm, ZPR64, ZZ_d>; 6704} 6705 6706class sve_int_perm_rev<bits<2> sz8_64, bits<2> opc, string asm, 6707 ZPRRegOp zprty> 6708: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, zprty:$Zn), 6709 asm, "\t$Zd, $Pg/m, $Zn", 6710 "", 6711 []>, Sched<[]> { 6712 bits<5> Zd; 6713 bits<3> Pg; 6714 bits<5> Zn; 6715 let Inst{31-24} = 0b00000101; 6716 let Inst{23-22} = sz8_64; 6717 let Inst{21-18} = 0b1001; 6718 let Inst{17-16} = opc; 6719 let Inst{15-13} = 0b100; 6720 let Inst{12-10} = Pg; 6721 let Inst{9-5} = Zn; 6722 let Inst{4-0} = Zd; 6723 6724 let Constraints = "$Zd = $_Zd"; 6725 let DestructiveInstType = DestructiveOther; 6726 let ElementSize = zprty.ElementSize; 6727} 6728 6729multiclass sve_int_perm_rev_rbit<string asm, SDPatternOperator op> { 6730 def _B : sve_int_perm_rev<0b00, 0b11, asm, ZPR8>; 6731 def _H : sve_int_perm_rev<0b01, 0b11, asm, ZPR16>; 6732 def _S : sve_int_perm_rev<0b10, 0b11, asm, ZPR32>; 6733 def _D : sve_int_perm_rev<0b11, 0b11, asm, ZPR64>; 6734 6735 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 6736 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6737 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6738 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6739} 6740 6741multiclass sve_int_perm_rev_revb<string asm, SDPatternOperator op> { 6742 def _H : sve_int_perm_rev<0b01, 0b00, asm, ZPR16>; 6743 def _S : sve_int_perm_rev<0b10, 0b00, asm, ZPR32>; 6744 def _D : sve_int_perm_rev<0b11, 0b00, asm, ZPR64>; 6745 6746 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 6747 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6748 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6749} 6750 6751multiclass sve_int_perm_rev_revh<string asm, SDPatternOperator op> { 6752 def _S : sve_int_perm_rev<0b10, 0b01, asm, ZPR32>; 6753 def _D : sve_int_perm_rev<0b11, 0b01, asm, ZPR64>; 6754 6755 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6756 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6757} 6758 6759multiclass sve_int_perm_rev_revw<string asm, SDPatternOperator op> { 6760 def _D : sve_int_perm_rev<0b11, 0b10, asm, ZPR64>; 6761 6762 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6763} 6764 6765class sve_int_perm_cpy_r<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6766 RegisterClass srcRegType> 6767: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegType:$Rn), 6768 asm, "\t$Zd, $Pg/m, $Rn", 6769 "", 6770 []>, Sched<[]> { 6771 bits<3> Pg; 6772 bits<5> Rn; 6773 bits<5> Zd; 6774 let Inst{31-24} = 0b00000101; 6775 let Inst{23-22} = sz8_64; 6776 let Inst{21-13} = 0b101000101; 6777 let Inst{12-10} = Pg; 6778 let Inst{9-5} = Rn; 6779 let Inst{4-0} = Zd; 6780 6781 let Constraints = "$Zd = $_Zd"; 6782 let DestructiveInstType = DestructiveOther; 6783 let ElementSize = zprty.ElementSize; 6784} 6785 6786multiclass sve_int_perm_cpy_r<string asm, SDPatternOperator op> { 6787 def _B : sve_int_perm_cpy_r<0b00, asm, ZPR8, GPR32sp>; 6788 def _H : sve_int_perm_cpy_r<0b01, asm, ZPR16, GPR32sp>; 6789 def _S : sve_int_perm_cpy_r<0b10, asm, ZPR32, GPR32sp>; 6790 def _D : sve_int_perm_cpy_r<0b11, asm, ZPR64, GPR64sp>; 6791 6792 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6793 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6794 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6795 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6796 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6797 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, GPR32sp:$Rn), 1>; 6798 def : InstAlias<"mov $Zd, $Pg/m, $Rn", 6799 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, GPR64sp:$Rn), 1>; 6800 6801 def : Pat<(nxv16i8 (op nxv16i1:$pg, i32:$splat, nxv16i8:$passthru)), 6802 (!cast<Instruction>(NAME # _B) $passthru, $pg, $splat)>; 6803 def : Pat<(nxv8i16 (op nxv8i1:$pg, i32:$splat, nxv8i16:$passthru)), 6804 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6805 def : Pat<(nxv4i32 (op nxv4i1:$pg, i32:$splat, nxv4i32:$passthru)), 6806 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6807 def : Pat<(nxv2i64 (op nxv2i1:$pg, i64:$splat, nxv2i64:$passthru)), 6808 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6809} 6810 6811class sve_int_perm_cpy_v<bits<2> sz8_64, string asm, ZPRRegOp zprty, 6812 RegisterClass srcRegtype> 6813: I<(outs zprty:$Zd), (ins zprty:$_Zd, PPR3bAny:$Pg, srcRegtype:$Vn), 6814 asm, "\t$Zd, $Pg/m, $Vn", 6815 "", 6816 []>, Sched<[]> { 6817 bits<3> Pg; 6818 bits<5> Vn; 6819 bits<5> Zd; 6820 let Inst{31-24} = 0b00000101; 6821 let Inst{23-22} = sz8_64; 6822 let Inst{21-13} = 0b100000100; 6823 let Inst{12-10} = Pg; 6824 let Inst{9-5} = Vn; 6825 let Inst{4-0} = Zd; 6826 6827 let Constraints = "$Zd = $_Zd"; 6828 let DestructiveInstType = DestructiveOther; 6829 let ElementSize = zprty.ElementSize; 6830} 6831 6832multiclass sve_int_perm_cpy_v<string asm, SDPatternOperator op> { 6833 def _B : sve_int_perm_cpy_v<0b00, asm, ZPR8, FPR8>; 6834 def _H : sve_int_perm_cpy_v<0b01, asm, ZPR16, FPR16>; 6835 def _S : sve_int_perm_cpy_v<0b10, asm, ZPR32, FPR32>; 6836 def _D : sve_int_perm_cpy_v<0b11, asm, ZPR64, FPR64>; 6837 6838 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6839 (!cast<Instruction>(NAME # _B) ZPR8:$Zd, PPR3bAny:$Pg, FPR8:$Vn), 1>; 6840 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6841 (!cast<Instruction>(NAME # _H) ZPR16:$Zd, PPR3bAny:$Pg, FPR16:$Vn), 1>; 6842 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6843 (!cast<Instruction>(NAME # _S) ZPR32:$Zd, PPR3bAny:$Pg, FPR32:$Vn), 1>; 6844 def : InstAlias<"mov $Zd, $Pg/m, $Vn", 6845 (!cast<Instruction>(NAME # _D) ZPR64:$Zd, PPR3bAny:$Pg, FPR64:$Vn), 1>; 6846 6847 def : Pat<(nxv8f16 (op nxv8i1:$pg, f16:$splat, nxv8f16:$passthru)), 6848 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6849 def : Pat<(nxv2f32 (op nxv2i1:$pg, f32:$splat, nxv2f32:$passthru)), 6850 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6851 def : Pat<(nxv4f32 (op nxv4i1:$pg, f32:$splat, nxv4f32:$passthru)), 6852 (!cast<Instruction>(NAME # _S) $passthru, $pg, $splat)>; 6853 def : Pat<(nxv2f64 (op nxv2i1:$pg, f64:$splat, nxv2f64:$passthru)), 6854 (!cast<Instruction>(NAME # _D) $passthru, $pg, $splat)>; 6855 6856 def : Pat<(nxv8bf16 (op nxv8i1:$pg, bf16:$splat, nxv8bf16:$passthru)), 6857 (!cast<Instruction>(NAME # _H) $passthru, $pg, $splat)>; 6858} 6859 6860class sve_int_perm_compact<bit sz, string asm, ZPRRegOp zprty> 6861: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn), 6862 asm, "\t$Zd, $Pg, $Zn", 6863 "", 6864 []>, Sched<[]> { 6865 bits<3> Pg; 6866 bits<5> Zd; 6867 bits<5> Zn; 6868 let Inst{31-23} = 0b000001011; 6869 let Inst{22} = sz; 6870 let Inst{21-13} = 0b100001100; 6871 let Inst{12-10} = Pg; 6872 let Inst{9-5} = Zn; 6873 let Inst{4-0} = Zd; 6874} 6875 6876multiclass sve_int_perm_compact<string asm, SDPatternOperator op> { 6877 def _S : sve_int_perm_compact<0b0, asm, ZPR32>; 6878 def _D : sve_int_perm_compact<0b1, asm, ZPR64>; 6879 6880 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 6881 def : SVE_2_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>; 6882 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 6883 def : SVE_2_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>; 6884} 6885 6886//===----------------------------------------------------------------------===// 6887// SVE Memory - Contiguous Load Group 6888//===----------------------------------------------------------------------===// 6889 6890class sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6891 RegisterOperand VecList> 6892: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6893 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6894 "", 6895 []>, Sched<[]> { 6896 bits<3> Pg; 6897 bits<5> Rn; 6898 bits<5> Zt; 6899 bits<4> imm4; 6900 let Inst{31-25} = 0b1010010; 6901 let Inst{24-21} = dtype; 6902 let Inst{20} = nf; 6903 let Inst{19-16} = imm4; 6904 let Inst{15-13} = 0b101; 6905 let Inst{12-10} = Pg; 6906 let Inst{9-5} = Rn; 6907 let Inst{4-0} = Zt; 6908 6909 let mayLoad = 1; 6910 let Uses = !if(nf, [FFR], []); 6911 let Defs = !if(nf, [FFR], []); 6912} 6913 6914multiclass sve_mem_cld_si_base<bits<4> dtype, bit nf, string asm, 6915 RegisterOperand listty, ZPRRegOp zprty> { 6916 def _REAL : sve_mem_cld_si_base<dtype, nf, asm, listty>; 6917 6918 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6919 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6920 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6921 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6922 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6923 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6924 6925 // We need a layer of indirection because early machine code passes balk at 6926 // physical register (i.e. FFR) uses that have no previous definition. 6927 let hasSideEffects = 1, hasNoSchedulingInfo = 1, mayLoad = 1 in { 6928 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), []>, 6929 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4)>; 6930 } 6931} 6932 6933multiclass sve_mem_cld_si<bits<4> dtype, string asm, RegisterOperand listty, 6934 ZPRRegOp zprty> 6935: sve_mem_cld_si_base<dtype, 0, asm, listty, zprty>; 6936 6937class sve_mem_cldnt_si_base<bits<2> msz, string asm, RegisterOperand VecList> 6938: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 6939 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6940 "", 6941 []>, Sched<[]> { 6942 bits<5> Zt; 6943 bits<3> Pg; 6944 bits<5> Rn; 6945 bits<4> imm4; 6946 let Inst{31-25} = 0b1010010; 6947 let Inst{24-23} = msz; 6948 let Inst{22-20} = 0b000; 6949 let Inst{19-16} = imm4; 6950 let Inst{15-13} = 0b111; 6951 let Inst{12-10} = Pg; 6952 let Inst{9-5} = Rn; 6953 let Inst{4-0} = Zt; 6954 6955 let mayLoad = 1; 6956} 6957 6958multiclass sve_mem_cldnt_si<bits<2> msz, string asm, RegisterOperand listty, 6959 ZPRRegOp zprty> { 6960 def NAME : sve_mem_cldnt_si_base<msz, asm, listty>; 6961 6962 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6963 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 6964 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 6965 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 6966 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 6967 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 6968} 6969 6970class sve_mem_cldnt_ss_base<bits<2> msz, string asm, RegisterOperand VecList, 6971 RegisterOperand gprty> 6972: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 6973 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 6974 "", 6975 []>, Sched<[]> { 6976 bits<3> Pg; 6977 bits<5> Rm; 6978 bits<5> Rn; 6979 bits<5> Zt; 6980 let Inst{31-25} = 0b1010010; 6981 let Inst{24-23} = msz; 6982 let Inst{22-21} = 0b00; 6983 let Inst{20-16} = Rm; 6984 let Inst{15-13} = 0b110; 6985 let Inst{12-10} = Pg; 6986 let Inst{9-5} = Rn; 6987 let Inst{4-0} = Zt; 6988 6989 let mayLoad = 1; 6990} 6991 6992multiclass sve_mem_cldnt_ss<bits<2> msz, string asm, RegisterOperand listty, 6993 ZPRRegOp zprty, RegisterOperand gprty> { 6994 def NAME : sve_mem_cldnt_ss_base<msz, asm, listty, gprty>; 6995 6996 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 6997 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 6998} 6999 7000class sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand VecList> 7001: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 7002 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 7003 bits<5> Zt; 7004 bits<5> Rn; 7005 bits<3> Pg; 7006 bits<4> imm4; 7007 let Inst{31-25} = 0b1010010; 7008 let Inst{24-23} = sz; 7009 let Inst{22-20} = 0; 7010 let Inst{19-16} = imm4; 7011 let Inst{15-13} = 0b001; 7012 let Inst{12-10} = Pg; 7013 let Inst{9-5} = Rn; 7014 let Inst{4-0} = Zt; 7015 7016 let mayLoad = 1; 7017} 7018 7019multiclass sve_mem_ldqr_si<bits<2> sz, string asm, RegisterOperand listty, 7020 ZPRRegOp zprty> { 7021 def NAME : sve_mem_ldqr_si<sz, asm, listty>; 7022 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7023 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7024 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7025 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 7026 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 7027 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s16:$imm4), 0>; 7028} 7029 7030class sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand VecList, 7031 RegisterOperand gprty> 7032: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7033 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 7034 bits<5> Zt; 7035 bits<3> Pg; 7036 bits<5> Rn; 7037 bits<5> Rm; 7038 let Inst{31-25} = 0b1010010; 7039 let Inst{24-23} = sz; 7040 let Inst{22-21} = 0; 7041 let Inst{20-16} = Rm; 7042 let Inst{15-13} = 0; 7043 let Inst{12-10} = Pg; 7044 let Inst{9-5} = Rn; 7045 let Inst{4-0} = Zt; 7046 7047 let mayLoad = 1; 7048} 7049 7050multiclass sve_mem_ldqr_ss<bits<2> sz, string asm, RegisterOperand listty, 7051 ZPRRegOp zprty, RegisterOperand gprty> { 7052 def NAME : sve_mem_ldqr_ss<sz, asm, listty, gprty>; 7053 7054 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7055 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7056} 7057 7058class sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 7059 RegisterOperand VecList, Operand immtype> 7060: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 7061 asm, "\t$Zt, $Pg/z, [$Rn, $imm6]", 7062 "", 7063 []>, Sched<[]> { 7064 bits<3> Pg; 7065 bits<5> Rn; 7066 bits<5> Zt; 7067 bits<6> imm6; 7068 let Inst{31-25} = 0b1000010; 7069 let Inst{24-23} = dtypeh; 7070 let Inst{22} = 1; 7071 let Inst{21-16} = imm6; 7072 let Inst{15} = 0b1; 7073 let Inst{14-13} = dtypel; 7074 let Inst{12-10} = Pg; 7075 let Inst{9-5} = Rn; 7076 let Inst{4-0} = Zt; 7077 7078 let mayLoad = 1; 7079} 7080 7081multiclass sve_mem_ld_dup<bits<2> dtypeh, bits<2> dtypel, string asm, 7082 RegisterOperand zlistty, ZPRRegOp zprty, Operand immtype> { 7083 def NAME : sve_mem_ld_dup<dtypeh, dtypel, asm, zlistty, immtype>; 7084 7085 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7086 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 7087 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm6]", 7088 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm6), 0>; 7089 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7090 (!cast<Instruction>(NAME) zlistty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7091} 7092 7093class sve_mem_cld_ss_base<bits<4> dtype, bit ff, dag iops, string asm, 7094 RegisterOperand VecList> 7095: I<(outs VecList:$Zt), iops, 7096 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 7097 "", 7098 []>, Sched<[]> { 7099 bits<5> Zt; 7100 bits<3> Pg; 7101 bits<5> Rm; 7102 bits<5> Rn; 7103 let Inst{31-25} = 0b1010010; 7104 let Inst{24-21} = dtype; 7105 let Inst{20-16} = Rm; 7106 let Inst{15-14} = 0b01; 7107 let Inst{13} = ff; 7108 let Inst{12-10} = Pg; 7109 let Inst{9-5} = Rn; 7110 let Inst{4-0} = Zt; 7111 7112 let mayLoad = 1; 7113 let Uses = !if(ff, [FFR], []); 7114 let Defs = !if(ff, [FFR], []); 7115} 7116 7117multiclass sve_mem_cld_ss<bits<4> dtype, string asm, RegisterOperand listty, 7118 ZPRRegOp zprty, RegisterOperand gprty> { 7119 def "" : sve_mem_cld_ss_base<dtype, 0, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7120 asm, listty>; 7121 7122 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7123 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7124} 7125 7126multiclass sve_mem_cldff_ss<bits<4> dtype, string asm, RegisterOperand listty, 7127 ZPRRegOp zprty, RegisterOperand gprty> { 7128 def _REAL : sve_mem_cld_ss_base<dtype, 1, (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7129 asm, listty>; 7130 7131 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 7132 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 7133 7134 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7135 (!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 7136 7137 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7138 (!cast<Instruction>(NAME # _REAL) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 7139 7140 // We need a layer of indirection because early machine code passes balk at 7141 // physical register (i.e. FFR) uses that have no previous definition. 7142 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7143 def "" : Pseudo<(outs listty:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), []>, 7144 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm)>; 7145 } 7146} 7147 7148multiclass sve_mem_cldnf_si<bits<4> dtype, string asm, RegisterOperand listty, 7149 ZPRRegOp zprty> 7150: sve_mem_cld_si_base<dtype, 1, asm, listty, zprty>; 7151 7152class sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList, 7153 string asm, Operand immtype> 7154: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, immtype:$imm4), 7155 asm, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 7156 "", 7157 []>, Sched<[]> { 7158 bits<5> Zt; 7159 bits<3> Pg; 7160 bits<5> Rn; 7161 bits<4> imm4; 7162 let Inst{31-25} = 0b1010010; 7163 let Inst{24-23} = sz; 7164 let Inst{22-21} = nregs{1-0}; 7165 let Inst{20} = nregs{2}; 7166 let Inst{19-16} = imm4; 7167 let Inst{15-13} = 0b111; 7168 let Inst{12-10} = Pg; 7169 let Inst{9-5} = Rn; 7170 let Inst{4-0} = Zt; 7171 7172 let mayLoad = 1; 7173} 7174 7175multiclass sve_mem_eld_si<bits<2> sz, bits<3> nregs, RegisterOperand VecList, 7176 string asm, Operand immtype> { 7177 def NAME : sve_mem_eld_si<sz, nregs, VecList, asm, immtype>; 7178 7179 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 7180 (!cast<Instruction>(NAME) VecList:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7181} 7182 7183 7184class sve_mem_eld_ss<bits<2> sz, bits<3> nregs, RegisterOperand VecList, 7185 string asm, RegisterOperand gprty> 7186: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7187 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", 7188 "", 7189 []>, Sched<[]> { 7190 bits<3> Pg; 7191 bits<5> Rm; 7192 bits<5> Rn; 7193 bits<5> Zt; 7194 let Inst{31-25} = 0b1010010; 7195 let Inst{24-23} = sz; 7196 let Inst{22-21} = nregs{1-0}; 7197 let Inst{20-16} = Rm; 7198 let Inst{15} = 0b1; 7199 let Inst{14} = nregs{2}; 7200 let Inst{13} = 0b0; 7201 let Inst{12-10} = Pg; 7202 let Inst{9-5} = Rn; 7203 let Inst{4-0} = Zt; 7204 7205 let mayLoad = 1; 7206} 7207 7208//===----------------------------------------------------------------------===// 7209// SVE Memory - 32-bit Gather and Unsized Contiguous Group 7210//===----------------------------------------------------------------------===// 7211 7212// bit xs is '1' if offsets are signed 7213// bit scaled is '1' if the offsets are scaled 7214class sve_mem_32b_gld_sv<bits<4> opc, bit xs, bit scaled, string asm, 7215 RegisterOperand zprext> 7216: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7217 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7218 "", 7219 []>, Sched<[]> { 7220 bits<3> Pg; 7221 bits<5> Rn; 7222 bits<5> Zm; 7223 bits<5> Zt; 7224 let Inst{31-25} = 0b1000010; 7225 let Inst{24-23} = opc{3-2}; 7226 let Inst{22} = xs; 7227 let Inst{21} = scaled; 7228 let Inst{20-16} = Zm; 7229 let Inst{15} = 0b0; 7230 let Inst{14-13} = opc{1-0}; 7231 let Inst{12-10} = Pg; 7232 let Inst{9-5} = Rn; 7233 let Inst{4-0} = Zt; 7234 7235 let mayLoad = 1; 7236 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7237 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7238} 7239 7240multiclass sve_mem_32b_gld_sv_32_scaled<bits<4> opc, string asm, 7241 SDPatternOperator sxtw_op, 7242 SDPatternOperator uxtw_op, 7243 RegisterOperand sxtw_opnd, 7244 RegisterOperand uxtw_opnd, 7245 ValueType vt> { 7246 def _UXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 0, 1, asm, uxtw_opnd>; 7247 def _SXTW_SCALED_REAL : sve_mem_32b_gld_sv<opc, 1, 1, asm, sxtw_opnd>; 7248 7249 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7250 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7251 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7252 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7253 7254 // We need a layer of indirection because early machine code passes balk at 7255 // physical register (i.e. FFR) uses that have no previous definition. 7256 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7257 def _UXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7258 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7259 def _SXTW_SCALED : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7260 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7261 } 7262 7263 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7264 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7265 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$indices), vt)), 7266 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7267} 7268 7269multiclass sve_mem_32b_gld_vs_32_unscaled<bits<4> opc, string asm, 7270 SDPatternOperator sxtw_op, 7271 SDPatternOperator uxtw_op, 7272 RegisterOperand sxtw_opnd, 7273 RegisterOperand uxtw_opnd, 7274 ValueType vt> { 7275 def _UXTW_REAL : sve_mem_32b_gld_sv<opc, 0, 0, asm, uxtw_opnd>; 7276 def _SXTW_REAL : sve_mem_32b_gld_sv<opc, 1, 0, asm, sxtw_opnd>; 7277 7278 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7279 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7280 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7281 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR32:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7282 7283 // We need a layer of indirection because early machine code passes balk at 7284 // physical register (i.e. FFR) uses that have no previous definition. 7285 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7286 def _UXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7287 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7288 def _SXTW : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7289 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_s:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7290 } 7291 7292 def : Pat<(nxv4i32 (uxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7293 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7294 def : Pat<(nxv4i32 (sxtw_op (nxv4i1 PPR:$gp), GPR64sp:$base, (nxv4i32 ZPR:$offsets), vt)), 7295 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7296} 7297 7298 7299class sve_mem_32b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7300: I<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7301 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7302 "", 7303 []>, Sched<[]> { 7304 bits<3> Pg; 7305 bits<5> Zn; 7306 bits<5> Zt; 7307 bits<5> imm5; 7308 let Inst{31-25} = 0b1000010; 7309 let Inst{24-23} = opc{3-2}; 7310 let Inst{22-21} = 0b01; 7311 let Inst{20-16} = imm5; 7312 let Inst{15} = 0b1; 7313 let Inst{14-13} = opc{1-0}; 7314 let Inst{12-10} = Pg; 7315 let Inst{9-5} = Zn; 7316 let Inst{4-0} = Zt; 7317 7318 let mayLoad = 1; 7319 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7320 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7321} 7322 7323multiclass sve_mem_32b_gld_vi_32_ptrs<bits<4> opc, string asm, Operand imm_ty, 7324 SDPatternOperator op, ValueType vt> { 7325 def _IMM_REAL : sve_mem_32b_gld_vi<opc, asm, imm_ty>; 7326 7327 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7328 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 0>; 7329 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7330 (!cast<Instruction>(NAME # _IMM_REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 0>; 7331 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7332 (!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7333 7334 // We need a layer of indirection because early machine code passes balk at 7335 // physical register (i.e. FFR) uses that have no previous definition. 7336 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7337 def _IMM : Pseudo<(outs Z_s:$Zt), (ins PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), []>, 7338 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5)>; 7339 } 7340 7341 def : Pat<(nxv4i32 (op (nxv4i1 PPR:$gp), (nxv4i32 ZPR:$ptrs), imm_ty:$index, vt)), 7342 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7343} 7344 7345class sve_mem_prfm_si<bits<2> msz, string asm> 7346: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, simm6s1:$imm6), 7347 asm, "\t$prfop, $Pg, [$Rn, $imm6, mul vl]", 7348 "", 7349 []>, Sched<[]> { 7350 bits<5> Rn; 7351 bits<3> Pg; 7352 bits<6> imm6; 7353 bits<4> prfop; 7354 let Inst{31-22} = 0b1000010111; 7355 let Inst{21-16} = imm6; 7356 let Inst{15} = 0b0; 7357 let Inst{14-13} = msz; 7358 let Inst{12-10} = Pg; 7359 let Inst{9-5} = Rn; 7360 let Inst{4} = 0b0; 7361 let Inst{3-0} = prfop; 7362 7363 let hasSideEffects = 1; 7364} 7365 7366multiclass sve_mem_prfm_si<bits<2> msz, string asm> { 7367 def NAME : sve_mem_prfm_si<msz, asm>; 7368 7369 def : InstAlias<asm # "\t$prfop, $Pg, [$Rn]", 7370 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 7371} 7372 7373class sve_mem_prfm_ss<bits<3> opc, string asm, RegisterOperand gprty> 7374: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 7375 asm, "\t$prfop, $Pg, [$Rn, $Rm]", 7376 "", 7377 []>, Sched<[]> { 7378 bits<5> Rm; 7379 bits<5> Rn; 7380 bits<3> Pg; 7381 bits<4> prfop; 7382 let Inst{31-25} = 0b1000010; 7383 let Inst{24-23} = opc{2-1}; 7384 let Inst{22-21} = 0b00; 7385 let Inst{20-16} = Rm; 7386 let Inst{15} = 0b1; 7387 let Inst{14} = opc{0}; 7388 let Inst{13} = 0b0; 7389 let Inst{12-10} = Pg; 7390 let Inst{9-5} = Rn; 7391 let Inst{4} = 0b0; 7392 let Inst{3-0} = prfop; 7393 7394 let hasSideEffects = 1; 7395} 7396 7397class sve_mem_32b_prfm_sv<bits<2> msz, bit xs, string asm, 7398 RegisterOperand zprext> 7399: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7400 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7401 "", 7402 []>, Sched<[]> { 7403 bits<3> Pg; 7404 bits<5> Rn; 7405 bits<5> Zm; 7406 bits<4> prfop; 7407 let Inst{31-23} = 0b100001000; 7408 let Inst{22} = xs; 7409 let Inst{21} = 0b1; 7410 let Inst{20-16} = Zm; 7411 let Inst{15} = 0b0; 7412 let Inst{14-13} = msz; 7413 let Inst{12-10} = Pg; 7414 let Inst{9-5} = Rn; 7415 let Inst{4} = 0b0; 7416 let Inst{3-0} = prfop; 7417 7418 let hasSideEffects = 1; 7419} 7420 7421multiclass sve_mem_32b_prfm_sv_scaled<bits<2> msz, string asm, 7422 RegisterOperand sxtw_opnd, 7423 RegisterOperand uxtw_opnd, 7424 SDPatternOperator op_sxtw, 7425 SDPatternOperator op_uxtw> { 7426 def _UXTW_SCALED : sve_mem_32b_prfm_sv<msz, 0, asm, uxtw_opnd>; 7427 def _SXTW_SCALED : sve_mem_32b_prfm_sv<msz, 1, asm, sxtw_opnd>; 7428 7429 def : Pat<(op_uxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7430 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7431 7432 def : Pat<(op_sxtw (nxv4i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv4i32 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7433 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7434} 7435 7436class sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7437: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, imm_ty:$imm5), 7438 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7439 "", 7440 []>, Sched<[]> { 7441 bits<3> Pg; 7442 bits<5> Zn; 7443 bits<5> imm5; 7444 bits<4> prfop; 7445 let Inst{31-25} = 0b1000010; 7446 let Inst{24-23} = msz; 7447 let Inst{22-21} = 0b00; 7448 let Inst{20-16} = imm5; 7449 let Inst{15-13} = 0b111; 7450 let Inst{12-10} = Pg; 7451 let Inst{9-5} = Zn; 7452 let Inst{4} = 0b0; 7453 let Inst{3-0} = prfop; 7454} 7455 7456multiclass sve_mem_32b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7457 def NAME : sve_mem_32b_prfm_vi<msz, asm, imm_ty>; 7458 7459 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7460 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR32:$Zn, 0), 1>; 7461 7462 def : Pat<(op (nxv4i1 PPR_3b:$Pg), (nxv4i32 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7463 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7464} 7465 7466class sve_mem_z_fill<string asm> 7467: I<(outs ZPRAny:$Zt), (ins GPR64sp:$Rn, simm9:$imm9), 7468 asm, "\t$Zt, [$Rn, $imm9, mul vl]", 7469 "", 7470 []>, Sched<[]> { 7471 bits<5> Rn; 7472 bits<5> Zt; 7473 bits<9> imm9; 7474 let Inst{31-22} = 0b1000010110; 7475 let Inst{21-16} = imm9{8-3}; 7476 let Inst{15-13} = 0b010; 7477 let Inst{12-10} = imm9{2-0}; 7478 let Inst{9-5} = Rn; 7479 let Inst{4-0} = Zt; 7480 7481 let mayLoad = 1; 7482} 7483 7484multiclass sve_mem_z_fill<string asm> { 7485 def NAME : sve_mem_z_fill<asm>; 7486 7487 def : InstAlias<asm # "\t$Zt, [$Rn]", 7488 (!cast<Instruction>(NAME) ZPRAny:$Zt, GPR64sp:$Rn, 0), 1>; 7489} 7490 7491class sve_mem_p_fill<string asm> 7492: I<(outs PPRAny:$Pt), (ins GPR64sp:$Rn, simm9:$imm9), 7493 asm, "\t$Pt, [$Rn, $imm9, mul vl]", 7494 "", 7495 []>, Sched<[]> { 7496 bits<4> Pt; 7497 bits<5> Rn; 7498 bits<9> imm9; 7499 let Inst{31-22} = 0b1000010110; 7500 let Inst{21-16} = imm9{8-3}; 7501 let Inst{15-13} = 0b000; 7502 let Inst{12-10} = imm9{2-0}; 7503 let Inst{9-5} = Rn; 7504 let Inst{4} = 0b0; 7505 let Inst{3-0} = Pt; 7506 7507 let mayLoad = 1; 7508} 7509 7510multiclass sve_mem_p_fill<string asm> { 7511 def NAME : sve_mem_p_fill<asm>; 7512 7513 def : InstAlias<asm # "\t$Pt, [$Rn]", 7514 (!cast<Instruction>(NAME) PPRAny:$Pt, GPR64sp:$Rn, 0), 1>; 7515} 7516 7517class sve2_mem_gldnt_vs_base<bits<5> opc, dag iops, string asm, 7518 RegisterOperand VecList> 7519: I<(outs VecList:$Zt), iops, 7520 asm, "\t$Zt, $Pg/z, [$Zn, $Rm]", 7521 "", 7522 []>, Sched<[]> { 7523 bits<3> Pg; 7524 bits<5> Rm; 7525 bits<5> Zn; 7526 bits<5> Zt; 7527 let Inst{31} = 0b1; 7528 let Inst{30} = opc{4}; 7529 let Inst{29-25} = 0b00010; 7530 let Inst{24-23} = opc{3-2}; 7531 let Inst{22-21} = 0b00; 7532 let Inst{20-16} = Rm; 7533 let Inst{15} = 0b1; 7534 let Inst{14-13} = opc{1-0}; 7535 let Inst{12-10} = Pg; 7536 let Inst{9-5} = Zn; 7537 let Inst{4-0} = Zt; 7538 7539 let mayLoad = 1; 7540} 7541 7542multiclass sve2_mem_gldnt_vs_32_ptrs<bits<5> opc, string asm, 7543 SDPatternOperator op, 7544 ValueType vt> { 7545 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 7546 asm, Z_s>; 7547 7548 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7549 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, GPR64:$Rm), 0>; 7550 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7551 (!cast<Instruction>(NAME # _REAL) ZPR32:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 0>; 7552 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7553 (!cast<Instruction>(NAME # _REAL) Z_s:$Zt, PPR3bAny:$Pg, ZPR32:$Zn, XZR), 1>; 7554 7555 def : Pat <(nxv4i32 (op (nxv4i1 PPR3bAny:$Pg), (nxv4i32 ZPR32:$Zd), (i64 GPR64:$Rm), vt)), 7556 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR32:$Zd, GPR64:$Rm)>; 7557} 7558 7559multiclass sve2_mem_gldnt_vs_64_ptrs<bits<5> opc, string asm, 7560 SDPatternOperator op, 7561 ValueType vt> { 7562 def _REAL : sve2_mem_gldnt_vs_base<opc, (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 7563 asm, Z_d>; 7564 7565 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $Rm]", 7566 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 0>; 7567 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7568 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 0>; 7569 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7570 (!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 7571 7572 def : Pat <(nxv2i64 (op (nxv2i1 PPR3bAny:$Pg), (nxv2i64 ZPR64:$Zd), (i64 GPR64:$Rm), vt)), 7573 (!cast<Instruction>(NAME # _REAL) PPR3bAny:$Pg, ZPR64:$Zd, GPR64:$Rm)>; 7574} 7575 7576//===----------------------------------------------------------------------===// 7577// SVE Memory - 64-bit Gather Group 7578//===----------------------------------------------------------------------===// 7579 7580// bit xs is '1' if offsets are signed 7581// bit scaled is '1' if the offsets are scaled 7582// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7583class sve_mem_64b_gld_sv<bits<4> opc, bit xs, bit scaled, bit lsl, string asm, 7584 RegisterOperand zprext> 7585: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7586 asm, "\t$Zt, $Pg/z, [$Rn, $Zm]", 7587 "", 7588 []>, Sched<[]> { 7589 bits<3> Pg; 7590 bits<5> Rn; 7591 bits<5> Zm; 7592 bits<5> Zt; 7593 let Inst{31-25} = 0b1100010; 7594 let Inst{24-23} = opc{3-2}; 7595 let Inst{22} = xs; 7596 let Inst{21} = scaled; 7597 let Inst{20-16} = Zm; 7598 let Inst{15} = lsl; 7599 let Inst{14-13} = opc{1-0}; 7600 let Inst{12-10} = Pg; 7601 let Inst{9-5} = Rn; 7602 let Inst{4-0} = Zt; 7603 7604 let mayLoad = 1; 7605 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7606 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7607} 7608 7609multiclass sve_mem_64b_gld_sv_32_scaled<bits<4> opc, string asm, 7610 SDPatternOperator sxtw_op, 7611 SDPatternOperator uxtw_op, 7612 RegisterOperand sxtw_opnd, 7613 RegisterOperand uxtw_opnd, 7614 ValueType vt> { 7615 def _UXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 0, 1, 0, asm, uxtw_opnd>; 7616 def _SXTW_SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 0, asm, sxtw_opnd>; 7617 7618 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7619 (!cast<Instruction>(NAME # _UXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7620 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7621 (!cast<Instruction>(NAME # _SXTW_SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7622 7623 // We need a layer of indirection because early machine code passes balk at 7624 // physical register (i.e. FFR) uses that have no previous definition. 7625 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7626 def _UXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7627 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7628 def _SXTW_SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7629 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7630 } 7631 7632 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7633 (!cast<Instruction>(NAME # _UXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7634 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7635 (!cast<Instruction>(NAME # _SXTW_SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7636} 7637 7638multiclass sve_mem_64b_gld_vs_32_unscaled<bits<4> opc, string asm, 7639 SDPatternOperator sxtw_op, 7640 SDPatternOperator uxtw_op, 7641 RegisterOperand sxtw_opnd, 7642 RegisterOperand uxtw_opnd, 7643 ValueType vt> { 7644 def _UXTW_REAL : sve_mem_64b_gld_sv<opc, 0, 0, 0, asm, uxtw_opnd>; 7645 def _SXTW_REAL : sve_mem_64b_gld_sv<opc, 1, 0, 0, asm, sxtw_opnd>; 7646 7647 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7648 (!cast<Instruction>(NAME # _UXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), 0>; 7649 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7650 (!cast<Instruction>(NAME # _SXTW_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), 0>; 7651 7652 // We need a layer of indirection because early machine code passes balk at 7653 // physical register (i.e. FFR) uses that have no previous definition. 7654 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7655 def _UXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm), []>, 7656 PseudoInstExpansion<(!cast<Instruction>(NAME # _UXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7657 def _SXTW : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm), []>, 7658 PseudoInstExpansion<(!cast<Instruction>(NAME # _SXTW_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7659 } 7660 7661 def : Pat<(nxv2i64 (uxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7662 (!cast<Instruction>(NAME # _UXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7663 def : Pat<(nxv2i64 (sxtw_op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7664 (!cast<Instruction>(NAME # _SXTW) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7665} 7666 7667multiclass sve_mem_64b_gld_sv2_64_scaled<bits<4> opc, string asm, 7668 SDPatternOperator op, 7669 RegisterOperand zprext, ValueType vt> { 7670 def _SCALED_REAL : sve_mem_64b_gld_sv<opc, 1, 1, 1, asm, zprext>; 7671 7672 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7673 (!cast<Instruction>(NAME # _SCALED_REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 0>; 7674 7675 // We need a layer of indirection because early machine code passes balk at 7676 // physical register (i.e. FFR) uses that have no previous definition. 7677 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7678 def _SCALED : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), []>, 7679 PseudoInstExpansion<(!cast<Instruction>(NAME # _SCALED_REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7680 } 7681 7682 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$indices), vt)), 7683 (!cast<Instruction>(NAME # _SCALED) PPR:$gp, GPR64sp:$base, ZPR:$indices)>; 7684} 7685 7686multiclass sve_mem_64b_gld_vs2_64_unscaled<bits<4> opc, string asm, 7687 SDPatternOperator op, ValueType vt> { 7688 def _REAL : sve_mem_64b_gld_sv<opc, 1, 0, 1, asm, ZPR64ExtLSL8>; 7689 7690 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Zm]", 7691 (!cast<Instruction>(NAME # _REAL) ZPR64:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), 0>; 7692 7693 // We need a layer of indirection because early machine code passes balk at 7694 // physical register (i.e. FFR) uses that have no previous definition. 7695 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7696 def "" : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm), []>, 7697 PseudoInstExpansion<(!cast<Instruction>(NAME # _REAL) Z_d:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, ZPR64ExtLSL8:$Zm)>; 7698 } 7699 7700 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), GPR64sp:$base, (nxv2i64 ZPR:$offsets), vt)), 7701 (!cast<Instruction>(NAME) PPR:$gp, GPR64sp:$base, ZPR:$offsets)>; 7702} 7703 7704class sve_mem_64b_gld_vi<bits<4> opc, string asm, Operand imm_ty> 7705: I<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7706 asm, "\t$Zt, $Pg/z, [$Zn, $imm5]", 7707 "", 7708 []>, Sched<[]> { 7709 bits<3> Pg; 7710 bits<5> Zn; 7711 bits<5> Zt; 7712 bits<5> imm5; 7713 let Inst{31-25} = 0b1100010; 7714 let Inst{24-23} = opc{3-2}; 7715 let Inst{22-21} = 0b01; 7716 let Inst{20-16} = imm5; 7717 let Inst{15} = 0b1; 7718 let Inst{14-13} = opc{1-0}; 7719 let Inst{12-10} = Pg; 7720 let Inst{9-5} = Zn; 7721 let Inst{4-0} = Zt; 7722 7723 let mayLoad = 1; 7724 let Defs = !if(!eq(opc{0}, 1), [FFR], []); 7725 let Uses = !if(!eq(opc{0}, 1), [FFR], []); 7726} 7727 7728multiclass sve_mem_64b_gld_vi_64_ptrs<bits<4> opc, string asm, Operand imm_ty, 7729 SDPatternOperator op, ValueType vt> { 7730 def _IMM_REAL : sve_mem_64b_gld_vi<opc, asm, imm_ty>; 7731 7732 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7733 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 0>; 7734 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn, $imm5]", 7735 (!cast<Instruction>(NAME # _IMM_REAL) ZPR64:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 0>; 7736 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Zn]", 7737 (!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7738 7739 // We need a layer of indirection because early machine code passes balk at 7740 // physical register (i.e. FFR) uses that have no previous definition. 7741 let hasSideEffects = 1, hasNoSchedulingInfo = 1 in { 7742 def _IMM : Pseudo<(outs Z_d:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), []>, 7743 PseudoInstExpansion<(!cast<Instruction>(NAME # _IMM_REAL) Z_d:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5)>; 7744 } 7745 7746 def : Pat<(nxv2i64 (op (nxv2i1 PPR:$gp), (nxv2i64 ZPR:$ptrs), imm_ty:$index, vt)), 7747 (!cast<Instruction>(NAME # _IMM) PPR:$gp, ZPR:$ptrs, imm_ty:$index)>; 7748} 7749 7750// bit lsl is '0' if the offsets are extended (uxtw/sxtw), '1' if shifted (lsl) 7751class sve_mem_64b_prfm_sv<bits<2> msz, bit xs, bit lsl, string asm, 7752 RegisterOperand zprext> 7753: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm), 7754 asm, "\t$prfop, $Pg, [$Rn, $Zm]", 7755 "", 7756 []>, Sched<[]> { 7757 bits<3> Pg; 7758 bits<5> Rn; 7759 bits<5> Zm; 7760 bits<4> prfop; 7761 let Inst{31-23} = 0b110001000; 7762 let Inst{22} = xs; 7763 let Inst{21} = 0b1; 7764 let Inst{20-16} = Zm; 7765 let Inst{15} = lsl; 7766 let Inst{14-13} = msz; 7767 let Inst{12-10} = Pg; 7768 let Inst{9-5} = Rn; 7769 let Inst{4} = 0b0; 7770 let Inst{3-0} = prfop; 7771 7772 let hasSideEffects = 1; 7773} 7774 7775multiclass sve_mem_64b_prfm_sv_ext_scaled<bits<2> msz, string asm, 7776 RegisterOperand sxtw_opnd, 7777 RegisterOperand uxtw_opnd, 7778 SDPatternOperator op_sxtw, 7779 SDPatternOperator op_uxtw> { 7780 def _UXTW_SCALED : sve_mem_64b_prfm_sv<msz, 0, 0, asm, uxtw_opnd>; 7781 def _SXTW_SCALED : sve_mem_64b_prfm_sv<msz, 1, 0, asm, sxtw_opnd>; 7782 7783 def : Pat<(op_uxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 uxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7784 (!cast<Instruction>(NAME # _UXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, uxtw_opnd:$Zm)>; 7785 7786 def : Pat<(op_sxtw (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 sxtw_opnd:$Zm), (i32 sve_prfop:$prfop)), 7787 (!cast<Instruction>(NAME # _SXTW_SCALED) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, sxtw_opnd:$Zm)>; 7788 7789} 7790 7791multiclass sve_mem_64b_prfm_sv_lsl_scaled<bits<2> msz, string asm, 7792 RegisterOperand zprext, SDPatternOperator frag> { 7793 def NAME : sve_mem_64b_prfm_sv<msz, 1, 1, asm, zprext>; 7794 7795 def : Pat<(frag (nxv2i1 PPR3bAny:$Pg), (i64 GPR64sp:$Rn), (nxv2i64 zprext:$Zm), (i32 sve_prfop:$prfop)), 7796 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, GPR64sp:$Rn, zprext:$Zm)>; 7797 7798} 7799 7800class sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty> 7801: I<(outs), (ins sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, imm_ty:$imm5), 7802 asm, "\t$prfop, $Pg, [$Zn, $imm5]", 7803 "", 7804 []>, Sched<[]> { 7805 bits<3> Pg; 7806 bits<5> Zn; 7807 bits<5> imm5; 7808 bits<4> prfop; 7809 let Inst{31-25} = 0b1100010; 7810 let Inst{24-23} = msz; 7811 let Inst{22-21} = 0b00; 7812 let Inst{20-16} = imm5; 7813 let Inst{15-13} = 0b111; 7814 let Inst{12-10} = Pg; 7815 let Inst{9-5} = Zn; 7816 let Inst{4} = 0b0; 7817 let Inst{3-0} = prfop; 7818 7819 let hasSideEffects = 1; 7820} 7821 7822multiclass sve_mem_64b_prfm_vi<bits<2> msz, string asm, Operand imm_ty, SDPatternOperator op> { 7823 def NAME : sve_mem_64b_prfm_vi<msz, asm, imm_ty>; 7824 7825 def : InstAlias<asm # "\t$prfop, $Pg, [$Zn]", 7826 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR3bAny:$Pg, ZPR64:$Zn, 0), 1>; 7827 7828 def : Pat<(op (nxv2i1 PPR_3b:$Pg), (nxv2i64 ZPR32:$Zn), (i64 imm_ty:$imm), (i32 sve_prfop:$prfop)), 7829 (!cast<Instruction>(NAME) sve_prfop:$prfop, PPR_3b:$Pg, ZPR32:$Zn, imm_ty:$imm)>; 7830} 7831 7832//===----------------------------------------------------------------------===// 7833// SVE Compute Vector Address Group 7834//===----------------------------------------------------------------------===// 7835 7836class sve_int_bin_cons_misc_0_a<bits<2> opc, bits<2> msz, string asm, 7837 ZPRRegOp zprty, RegisterOperand zprext> 7838: I<(outs zprty:$Zd), (ins zprty:$Zn, zprext:$Zm), 7839 asm, "\t$Zd, [$Zn, $Zm]", 7840 "", 7841 []>, Sched<[]> { 7842 bits<5> Zd; 7843 bits<5> Zn; 7844 bits<5> Zm; 7845 let Inst{31-24} = 0b00000100; 7846 let Inst{23-22} = opc; 7847 let Inst{21} = 0b1; 7848 let Inst{20-16} = Zm; 7849 let Inst{15-12} = 0b1010; 7850 let Inst{11-10} = msz; 7851 let Inst{9-5} = Zn; 7852 let Inst{4-0} = Zd; 7853} 7854 7855multiclass sve_int_bin_cons_misc_0_a_uxtw<bits<2> opc, string asm> { 7856 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtUXTW8>; 7857 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtUXTW16>; 7858 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtUXTW32>; 7859 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtUXTW64>; 7860} 7861 7862multiclass sve_int_bin_cons_misc_0_a_sxtw<bits<2> opc, string asm> { 7863 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtSXTW8>; 7864 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtSXTW16>; 7865 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtSXTW32>; 7866 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtSXTW64>; 7867} 7868 7869multiclass sve_int_bin_cons_misc_0_a_32_lsl<bits<2> opc, string asm> { 7870 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR32, ZPR32ExtLSL8>; 7871 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR32, ZPR32ExtLSL16>; 7872 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR32, ZPR32ExtLSL32>; 7873 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR32, ZPR32ExtLSL64>; 7874} 7875 7876multiclass sve_int_bin_cons_misc_0_a_64_lsl<bits<2> opc, string asm> { 7877 def _0 : sve_int_bin_cons_misc_0_a<opc, 0b00, asm, ZPR64, ZPR64ExtLSL8>; 7878 def _1 : sve_int_bin_cons_misc_0_a<opc, 0b01, asm, ZPR64, ZPR64ExtLSL16>; 7879 def _2 : sve_int_bin_cons_misc_0_a<opc, 0b10, asm, ZPR64, ZPR64ExtLSL32>; 7880 def _3 : sve_int_bin_cons_misc_0_a<opc, 0b11, asm, ZPR64, ZPR64ExtLSL64>; 7881} 7882 7883//===----------------------------------------------------------------------===// 7884// SVE Integer Misc - Unpredicated Group 7885//===----------------------------------------------------------------------===// 7886 7887class sve_int_bin_cons_misc_0_b<bits<2> sz, string asm, ZPRRegOp zprty> 7888: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 7889 asm, "\t$Zd, $Zn, $Zm", 7890 "", 7891 []>, Sched<[]> { 7892 bits<5> Zd; 7893 bits<5> Zm; 7894 bits<5> Zn; 7895 let Inst{31-24} = 0b00000100; 7896 let Inst{23-22} = sz; 7897 let Inst{21} = 0b1; 7898 let Inst{20-16} = Zm; 7899 let Inst{15-10} = 0b101100; 7900 let Inst{9-5} = Zn; 7901 let Inst{4-0} = Zd; 7902} 7903 7904multiclass sve_int_bin_cons_misc_0_b<string asm, SDPatternOperator op> { 7905 def _H : sve_int_bin_cons_misc_0_b<0b01, asm, ZPR16>; 7906 def _S : sve_int_bin_cons_misc_0_b<0b10, asm, ZPR32>; 7907 def _D : sve_int_bin_cons_misc_0_b<0b11, asm, ZPR64>; 7908 7909 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8i16, !cast<Instruction>(NAME # _H)>; 7910 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4i32, !cast<Instruction>(NAME # _S)>; 7911 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2i64, !cast<Instruction>(NAME # _D)>; 7912} 7913 7914class sve_int_bin_cons_misc_0_c<bits<8> opc, string asm, ZPRRegOp zprty> 7915: I<(outs zprty:$Zd), (ins zprty:$Zn), 7916 asm, "\t$Zd, $Zn", 7917 "", 7918 []>, Sched<[]> { 7919 bits<5> Zd; 7920 bits<5> Zn; 7921 let Inst{31-24} = 0b00000100; 7922 let Inst{23-22} = opc{7-6}; 7923 let Inst{21} = 0b1; 7924 let Inst{20-16} = opc{5-1}; 7925 let Inst{15-11} = 0b10111; 7926 let Inst{10} = opc{0}; 7927 let Inst{9-5} = Zn; 7928 let Inst{4-0} = Zd; 7929 7930 let hasSideEffects = 0; 7931} 7932 7933multiclass sve_int_bin_cons_misc_0_c_fexpa<string asm, SDPatternOperator op> { 7934 def _H : sve_int_bin_cons_misc_0_c<0b01000000, asm, ZPR16>; 7935 def _S : sve_int_bin_cons_misc_0_c<0b10000000, asm, ZPR32>; 7936 def _D : sve_int_bin_cons_misc_0_c<0b11000000, asm, ZPR64>; 7937 7938 def : SVE_1_Op_Pat<nxv8f16, op, nxv8i16, !cast<Instruction>(NAME # _H)>; 7939 def : SVE_1_Op_Pat<nxv4f32, op, nxv4i32, !cast<Instruction>(NAME # _S)>; 7940 def : SVE_1_Op_Pat<nxv2f64, op, nxv2i64, !cast<Instruction>(NAME # _D)>; 7941} 7942 7943//===----------------------------------------------------------------------===// 7944// SVE Integer Reduction Group 7945//===----------------------------------------------------------------------===// 7946 7947class sve_int_reduce<bits<2> sz8_32, bits<2> fmt, bits<3> opc, string asm, 7948 ZPRRegOp zprty, FPRasZPROperand dstOpType> 7949: I<(outs dstOpType:$Vd), (ins PPR3bAny:$Pg, zprty:$Zn), 7950 asm, "\t$Vd, $Pg, $Zn", 7951 "", 7952 []>, Sched<[]> { 7953 bits<3> Pg; 7954 bits<5> Vd; 7955 bits<5> Zn; 7956 let Inst{31-24} = 0b00000100; 7957 let Inst{23-22} = sz8_32; 7958 let Inst{21} = 0b0; 7959 let Inst{20-19} = fmt; 7960 let Inst{18-16} = opc; 7961 let Inst{15-13} = 0b001; 7962 let Inst{12-10} = Pg; 7963 let Inst{9-5} = Zn; 7964 let Inst{4-0} = Vd; 7965} 7966 7967multiclass sve_int_reduce_0_saddv<bits<3> opc, string asm, 7968 SDPatternOperator op> { 7969 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7970 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7971 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7972 7973 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7974 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7975 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7976} 7977 7978multiclass sve_int_reduce_0_uaddv<bits<3> opc, string asm, 7979 SDPatternOperator op> { 7980 def _B : sve_int_reduce<0b00, 0b00, opc, asm, ZPR8, FPR64asZPR>; 7981 def _H : sve_int_reduce<0b01, 0b00, opc, asm, ZPR16, FPR64asZPR>; 7982 def _S : sve_int_reduce<0b10, 0b00, opc, asm, ZPR32, FPR64asZPR>; 7983 def _D : sve_int_reduce<0b11, 0b00, opc, asm, ZPR64, FPR64asZPR>; 7984 7985 def : SVE_2_Op_Pat<nxv2i64, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7986 def : SVE_2_Op_Pat<nxv2i64, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 7987 def : SVE_2_Op_Pat<nxv2i64, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 7988 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 7989} 7990 7991multiclass sve_int_reduce_1<bits<3> opc, string asm, 7992 SDPatternOperator op> { 7993 def _B : sve_int_reduce<0b00, 0b01, opc, asm, ZPR8, FPR8asZPR>; 7994 def _H : sve_int_reduce<0b01, 0b01, opc, asm, ZPR16, FPR16asZPR>; 7995 def _S : sve_int_reduce<0b10, 0b01, opc, asm, ZPR32, FPR32asZPR>; 7996 def _D : sve_int_reduce<0b11, 0b01, opc, asm, ZPR64, FPR64asZPR>; 7997 7998 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 7999 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 8000 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 8001 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 8002} 8003 8004multiclass sve_int_reduce_2<bits<3> opc, string asm, 8005 SDPatternOperator op> { 8006 def _B : sve_int_reduce<0b00, 0b11, opc, asm, ZPR8, FPR8asZPR>; 8007 def _H : sve_int_reduce<0b01, 0b11, opc, asm, ZPR16, FPR16asZPR>; 8008 def _S : sve_int_reduce<0b10, 0b11, opc, asm, ZPR32, FPR32asZPR>; 8009 def _D : sve_int_reduce<0b11, 0b11, opc, asm, ZPR64, FPR64asZPR>; 8010 8011 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>; 8012 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>; 8013 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>; 8014 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>; 8015} 8016 8017class sve_int_movprfx_pred<bits<2> sz8_32, bits<3> opc, string asm, 8018 ZPRRegOp zprty, string pg_suffix, dag iops> 8019: I<(outs zprty:$Zd), iops, 8020 asm, "\t$Zd, $Pg"#pg_suffix#", $Zn", 8021 "", 8022 []>, Sched<[]> { 8023 bits<3> Pg; 8024 bits<5> Zd; 8025 bits<5> Zn; 8026 let Inst{31-24} = 0b00000100; 8027 let Inst{23-22} = sz8_32; 8028 let Inst{21-19} = 0b010; 8029 let Inst{18-16} = opc; 8030 let Inst{15-13} = 0b001; 8031 let Inst{12-10} = Pg; 8032 let Inst{9-5} = Zn; 8033 let Inst{4-0} = Zd; 8034 8035 let ElementSize = zprty.ElementSize; 8036} 8037 8038multiclass sve_int_movprfx_pred_merge<bits<3> opc, string asm> { 8039let Constraints = "$Zd = $_Zd" in { 8040 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/m", 8041 (ins ZPR8:$_Zd, PPR3bAny:$Pg, ZPR8:$Zn)>; 8042 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/m", 8043 (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR16:$Zn)>; 8044 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/m", 8045 (ins ZPR32:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn)>; 8046 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/m", 8047 (ins ZPR64:$_Zd, PPR3bAny:$Pg, ZPR64:$Zn)>; 8048} 8049} 8050 8051multiclass sve_int_movprfx_pred_zero<bits<3> opc, string asm> { 8052 def _B : sve_int_movprfx_pred<0b00, opc, asm, ZPR8, "/z", 8053 (ins PPR3bAny:$Pg, ZPR8:$Zn)>; 8054 def _H : sve_int_movprfx_pred<0b01, opc, asm, ZPR16, "/z", 8055 (ins PPR3bAny:$Pg, ZPR16:$Zn)>; 8056 def _S : sve_int_movprfx_pred<0b10, opc, asm, ZPR32, "/z", 8057 (ins PPR3bAny:$Pg, ZPR32:$Zn)>; 8058 def _D : sve_int_movprfx_pred<0b11, opc, asm, ZPR64, "/z", 8059 (ins PPR3bAny:$Pg, ZPR64:$Zn)>; 8060} 8061 8062//===----------------------------------------------------------------------===// 8063// SVE Propagate Break Group 8064//===----------------------------------------------------------------------===// 8065 8066class sve_int_brkp<bits<2> opc, string asm> 8067: I<(outs PPR8:$Pd), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$Pm), 8068 asm, "\t$Pd, $Pg/z, $Pn, $Pm", 8069 "", 8070 []>, Sched<[]> { 8071 bits<4> Pd; 8072 bits<4> Pg; 8073 bits<4> Pm; 8074 bits<4> Pn; 8075 let Inst{31-24} = 0b00100101; 8076 let Inst{23} = 0b0; 8077 let Inst{22} = opc{1}; 8078 let Inst{21-20} = 0b00; 8079 let Inst{19-16} = Pm; 8080 let Inst{15-14} = 0b11; 8081 let Inst{13-10} = Pg; 8082 let Inst{9} = 0b0; 8083 let Inst{8-5} = Pn; 8084 let Inst{4} = opc{0}; 8085 let Inst{3-0} = Pd; 8086 8087 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 8088} 8089 8090multiclass sve_int_brkp<bits<2> opc, string asm, SDPatternOperator op> { 8091 def NAME : sve_int_brkp<opc, asm>; 8092 8093 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8094} 8095 8096 8097//===----------------------------------------------------------------------===// 8098// SVE Partition Break Group 8099//===----------------------------------------------------------------------===// 8100 8101class sve_int_brkn<bit S, string asm> 8102: I<(outs PPR8:$Pdm), (ins PPRAny:$Pg, PPR8:$Pn, PPR8:$_Pdm), 8103 asm, "\t$Pdm, $Pg/z, $Pn, $_Pdm", 8104 "", 8105 []>, Sched<[]> { 8106 bits<4> Pdm; 8107 bits<4> Pg; 8108 bits<4> Pn; 8109 let Inst{31-23} = 0b001001010; 8110 let Inst{22} = S; 8111 let Inst{21-14} = 0b01100001; 8112 let Inst{13-10} = Pg; 8113 let Inst{9} = 0b0; 8114 let Inst{8-5} = Pn; 8115 let Inst{4} = 0b0; 8116 let Inst{3-0} = Pdm; 8117 8118 let Constraints = "$Pdm = $_Pdm"; 8119 let Defs = !if(S, [NZCV], []); 8120 let ElementSize = ElementSizeB; 8121} 8122 8123multiclass sve_int_brkn<bits<1> opc, string asm, SDPatternOperator op> { 8124 def NAME : sve_int_brkn<opc, asm>; 8125 8126 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8127} 8128 8129class sve_int_break<bits<3> opc, string asm, string suffix, dag iops> 8130: I<(outs PPR8:$Pd), iops, 8131 asm, "\t$Pd, $Pg"#suffix#", $Pn", 8132 "", 8133 []>, Sched<[]> { 8134 bits<4> Pd; 8135 bits<4> Pg; 8136 bits<4> Pn; 8137 let Inst{31-24} = 0b00100101; 8138 let Inst{23-22} = opc{2-1}; 8139 let Inst{21-14} = 0b01000001; 8140 let Inst{13-10} = Pg; 8141 let Inst{9} = 0b0; 8142 let Inst{8-5} = Pn; 8143 let Inst{4} = opc{0}; 8144 let Inst{3-0} = Pd; 8145 8146 let Constraints = !if(!eq (opc{0}, 1), "$Pd = $_Pd", ""); 8147 let Defs = !if(!eq (opc{1}, 1), [NZCV], []); 8148 8149} 8150 8151multiclass sve_int_break_m<bits<3> opc, string asm, SDPatternOperator op> { 8152 def NAME : sve_int_break<opc, asm, "/m", (ins PPR8:$_Pd, PPRAny:$Pg, PPR8:$Pn)>; 8153 8154 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8155} 8156 8157multiclass sve_int_break_z<bits<3> opc, string asm, SDPatternOperator op> { 8158 def NAME : sve_int_break<opc, asm, "/z", (ins PPRAny:$Pg, PPR8:$Pn)>; 8159 8160 def : SVE_2_Op_Pat<nxv16i1, op, nxv16i1, nxv16i1, !cast<Instruction>(NAME)>; 8161} 8162 8163//===----------------------------------------------------------------------===// 8164// SVE2 String Processing Group 8165//===----------------------------------------------------------------------===// 8166 8167class sve2_char_match<bit sz, bit opc, string asm, 8168 PPRRegOp pprty, ZPRRegOp zprty> 8169: I<(outs pprty:$Pd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8170 asm, "\t$Pd, $Pg/z, $Zn, $Zm", 8171 "", 8172 []>, Sched<[]> { 8173 bits<4> Pd; 8174 bits<3> Pg; 8175 bits<5> Zm; 8176 bits<5> Zn; 8177 let Inst{31-23} = 0b010001010; 8178 let Inst{22} = sz; 8179 let Inst{21} = 0b1; 8180 let Inst{20-16} = Zm; 8181 let Inst{15-13} = 0b100; 8182 let Inst{12-10} = Pg; 8183 let Inst{9-5} = Zn; 8184 let Inst{4} = opc; 8185 let Inst{3-0} = Pd; 8186 8187 let Defs = [NZCV]; 8188 let ElementSize = pprty.ElementSize; 8189 let isPTestLike = 1; 8190} 8191 8192multiclass sve2_char_match<bit opc, string asm, SDPatternOperator op> { 8193 def _B : sve2_char_match<0b0, opc, asm, PPR8, ZPR8>; 8194 def _H : sve2_char_match<0b1, opc, asm, PPR16, ZPR16>; 8195 8196 def : SVE_3_Op_Pat<nxv16i1, op, nxv16i1, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 8197 def : SVE_3_Op_Pat<nxv8i1, op, nxv8i1, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 8198} 8199 8200//===----------------------------------------------------------------------===// 8201// SVE2 Histogram Computation - Segment Group 8202//===----------------------------------------------------------------------===// 8203 8204class sve2_hist_gen_segment<string asm, SDPatternOperator op> 8205: I<(outs ZPR8:$Zd), (ins ZPR8:$Zn, ZPR8:$Zm), 8206 asm, "\t$Zd, $Zn, $Zm", 8207 "", 8208 [(set nxv16i8:$Zd, (op nxv16i8:$Zn, nxv16i8:$Zm))]>, Sched<[]> { 8209 bits<5> Zd; 8210 bits<5> Zn; 8211 bits<5> Zm; 8212 let Inst{31-21} = 0b01000101001; 8213 let Inst{20-16} = Zm; 8214 let Inst{15-10} = 0b101000; 8215 let Inst{9-5} = Zn; 8216 let Inst{4-0} = Zd; 8217} 8218 8219//===----------------------------------------------------------------------===// 8220// SVE2 Histogram Computation - Vector Group 8221//===----------------------------------------------------------------------===// 8222 8223class sve2_hist_gen_vector<bit sz, string asm, ZPRRegOp zprty> 8224: I<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zn, zprty:$Zm), 8225 asm, "\t$Zd, $Pg/z, $Zn, $Zm", 8226 "", 8227 []>, Sched<[]> { 8228 bits<5> Zd; 8229 bits<5> Zn; 8230 bits<3> Pg; 8231 bits<5> Zm; 8232 let Inst{31-23} = 0b010001011; 8233 let Inst{22} = sz; 8234 let Inst{21} = 0b1; 8235 let Inst{20-16} = Zm; 8236 let Inst{15-13} = 0b110; 8237 let Inst{12-10} = Pg; 8238 let Inst{9-5} = Zn; 8239 let Inst{4-0} = Zd; 8240} 8241 8242multiclass sve2_hist_gen_vector<string asm, SDPatternOperator op> { 8243 def _S : sve2_hist_gen_vector<0b0, asm, ZPR32>; 8244 def _D : sve2_hist_gen_vector<0b1, asm, ZPR64>; 8245 8246 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 8247 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 8248} 8249 8250//===----------------------------------------------------------------------===// 8251// SVE2 Crypto Extensions Group 8252//===----------------------------------------------------------------------===// 8253 8254class sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty> 8255: I<(outs zprty:$Zd), (ins zprty:$Zn, zprty:$Zm), 8256 asm, "\t$Zd, $Zn, $Zm", 8257 "", 8258 []>, Sched<[]> { 8259 bits<5> Zd; 8260 bits<5> Zn; 8261 bits<5> Zm; 8262 let Inst{31-21} = 0b01000101001; 8263 let Inst{20-16} = Zm; 8264 let Inst{15-11} = 0b11110; 8265 let Inst{10} = opc; 8266 let Inst{9-5} = Zn; 8267 let Inst{4-0} = Zd; 8268} 8269 8270multiclass sve2_crypto_cons_bin_op<bit opc, string asm, ZPRRegOp zprty, 8271 SDPatternOperator op, ValueType vt> { 8272 def NAME : sve2_crypto_cons_bin_op<opc, asm, zprty>; 8273 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8274} 8275 8276class sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty> 8277: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, zprty:$Zm), 8278 asm, "\t$Zdn, $_Zdn, $Zm", 8279 "", 8280 []>, Sched<[]> { 8281 bits<5> Zdn; 8282 bits<5> Zm; 8283 let Inst{31-17} = 0b010001010010001; 8284 let Inst{16} = opc{1}; 8285 let Inst{15-11} = 0b11100; 8286 let Inst{10} = opc{0}; 8287 let Inst{9-5} = Zm; 8288 let Inst{4-0} = Zdn; 8289 8290 let Constraints = "$Zdn = $_Zdn"; 8291} 8292 8293multiclass sve2_crypto_des_bin_op<bits<2> opc, string asm, ZPRRegOp zprty, 8294 SDPatternOperator op, ValueType vt> { 8295 def NAME : sve2_crypto_des_bin_op<opc, asm, zprty>; 8296 def : SVE_2_Op_Pat<vt, op, vt, vt, !cast<Instruction>(NAME)>; 8297} 8298 8299class sve2_crypto_unary_op<bit opc, string asm, ZPRRegOp zprty> 8300: I<(outs zprty:$Zdn), (ins zprty:$_Zdn), 8301 asm, "\t$Zdn, $_Zdn", 8302 "", 8303 []>, Sched<[]> { 8304 bits<5> Zdn; 8305 let Inst{31-11} = 0b010001010010000011100; 8306 let Inst{10} = opc; 8307 let Inst{9-5} = 0b00000; 8308 let Inst{4-0} = Zdn; 8309 8310 let Constraints = "$Zdn = $_Zdn"; 8311} 8312 8313multiclass sve2_crypto_unary_op<bit opc, string asm, SDPatternOperator op> { 8314 def NAME : sve2_crypto_unary_op<opc, asm, ZPR8>; 8315 def : SVE_1_Op_Pat<nxv16i8, op, nxv16i8, !cast<Instruction>(NAME)>; 8316} 8317 8318//===----------------------------------------------------------------------===// 8319// SVE BFloat16 Group 8320//===----------------------------------------------------------------------===// 8321 8322class sve_float_dot<bit bf, string asm> 8323: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8324 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8325 bits<5> Zda; 8326 bits<5> Zn; 8327 bits<5> Zm; 8328 let Inst{31-23} = 0b011001000; 8329 let Inst{22} = bf; 8330 let Inst{21} = 0b1; 8331 let Inst{20-16} = Zm; 8332 let Inst{15-10} = 0b100000; 8333 let Inst{9-5} = Zn; 8334 let Inst{4-0} = Zda; 8335 8336 let Constraints = "$Zda = $_Zda"; 8337 let DestructiveInstType = DestructiveOther; 8338} 8339 8340multiclass sve_float_dot<bit bf, string asm, ValueType InVT, SDPatternOperator op> { 8341 def NAME : sve_float_dot<bf, asm>; 8342 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, InVT, InVT, !cast<Instruction>(NAME)>; 8343} 8344 8345class sve_float_dot_indexed<bit bf, string asm> 8346: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS32b:$iop), 8347 asm, "\t$Zda, $Zn, $Zm$iop", "", []>, Sched<[]> { 8348 bits<5> Zda; 8349 bits<5> Zn; 8350 bits<3> Zm; 8351 bits<2> iop; 8352 let Inst{31-23} = 0b011001000; 8353 let Inst{22} = bf; 8354 let Inst{21} = 0b1; 8355 let Inst{20-19} = iop; 8356 let Inst{18-16} = Zm; 8357 let Inst{15-10} = 0b010000; 8358 let Inst{9-5} = Zn; 8359 let Inst{4-0} = Zda; 8360 8361 let Constraints = "$Zda = $_Zda"; 8362 let DestructiveInstType = DestructiveOther; 8363} 8364 8365multiclass sve_float_dot_indexed<bit bf, string asm, ValueType InVT, SDPatternOperator op> { 8366 def NAME : sve_float_dot_indexed<bf, asm>; 8367 def : SVE_4_Op_Imm_Pat<nxv4f32, op, nxv4f32, InVT, InVT, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8368} 8369 8370class sve_bfloat_matmul<string asm> 8371: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8372 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8373 bits<5> Zm; 8374 bits<5> Zda; 8375 bits<5> Zn; 8376 let Inst{31-21} = 0b01100100011; 8377 let Inst{20-16} = Zm; 8378 let Inst{15-10} = 0b111001; 8379 let Inst{9-5} = Zn; 8380 let Inst{4-0} = Zda; 8381 8382 let Constraints = "$Zda = $_Zda"; 8383 let DestructiveInstType = DestructiveOther; 8384 let ElementSize = ElementSizeH; 8385} 8386 8387multiclass sve_bfloat_matmul<string asm, SDPatternOperator op> { 8388 def NAME : sve_bfloat_matmul<asm>; 8389 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv8bf16, nxv8bf16 ,!cast<Instruction>(NAME)>; 8390} 8391 8392class sve_bfloat_convert<bit N, string asm> 8393: I<(outs ZPR16:$Zd), (ins ZPR16:$_Zd, PPR3bAny:$Pg, ZPR32:$Zn), 8394 asm, "\t$Zd, $Pg/m, $Zn", "", []>, Sched<[]> { 8395 bits<5> Zd; 8396 bits<3> Pg; 8397 bits<5> Zn; 8398 let Inst{31-25} = 0b0110010; 8399 let Inst{24} = N; 8400 let Inst{23-13} = 0b10001010101; 8401 let Inst{12-10} = Pg; 8402 let Inst{9-5} = Zn; 8403 let Inst{4-0} = Zd; 8404 8405 let Constraints = "$Zd = $_Zd"; 8406 let DestructiveInstType = DestructiveOther; 8407 let hasSideEffects = 1; 8408 let ElementSize = ElementSizeS; 8409} 8410 8411multiclass sve_bfloat_convert<bit N, string asm, SDPatternOperator op> { 8412 def NAME : sve_bfloat_convert<N, asm>; 8413 def : SVE_3_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8i1, nxv4f32, !cast<Instruction>(NAME)>; 8414} 8415 8416//===----------------------------------------------------------------------===// 8417// SVE Integer Matrix Multiply Group 8418//===----------------------------------------------------------------------===// 8419 8420class sve_int_matmul<bits<2> uns, string asm> 8421: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8422 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8423 bits<5> Zda; 8424 bits<5> Zn; 8425 bits<5> Zm; 8426 let Inst{31-24} = 0b01000101; 8427 let Inst{23-22} = uns; 8428 let Inst{21} = 0; 8429 let Inst{20-16} = Zm; 8430 let Inst{15-10} = 0b100110; 8431 let Inst{9-5} = Zn; 8432 let Inst{4-0} = Zda; 8433 8434 let Constraints = "$Zda = $_Zda"; 8435 let DestructiveInstType = DestructiveOther; 8436 let ElementSize = ZPR32.ElementSize; 8437} 8438 8439multiclass sve_int_matmul<bits<2> uns, string asm, SDPatternOperator op> { 8440 def NAME : sve_int_matmul<uns, asm>; 8441 8442 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8443} 8444 8445//===----------------------------------------------------------------------===// 8446// SVE Integer Dot Product Mixed Sign Group 8447//===----------------------------------------------------------------------===// 8448 8449class sve_int_dot_mixed<string asm> 8450: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR8:$Zm), asm, 8451 "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8452 bits<5> Zda; 8453 bits<5> Zn; 8454 bits<5> Zm; 8455 let Inst{31-21} = 0b01000100100; 8456 let Inst{20-16} = Zm; 8457 let Inst{15-10} = 0b011110; 8458 let Inst{9-5} = Zn; 8459 let Inst{4-0} = Zda; 8460 8461 let Constraints = "$Zda = $_Zda"; 8462 let DestructiveInstType = DestructiveOther; 8463 let ElementSize = ZPR32.ElementSize; 8464} 8465 8466multiclass sve_int_dot_mixed<string asm, SDPatternOperator op> { 8467 def NAME : sve_int_dot_mixed<asm>; 8468 8469 def : SVE_3_Op_Pat<nxv4i32, op , nxv4i32, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8470} 8471 8472//===----------------------------------------------------------------------===// 8473// SVE Integer Dot Product Mixed Sign - Indexed Group 8474//===----------------------------------------------------------------------===// 8475 8476class sve_int_dot_mixed_indexed<bit U, string asm> 8477: I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR8:$Zn, ZPR3b8:$Zm, VectorIndexS32b:$idx), 8478 asm, "\t$Zda, $Zn, $Zm$idx", "", []>, Sched<[]> { 8479 bits<5> Zda; 8480 bits<5> Zn; 8481 bits<3> Zm; 8482 bits<2> idx; 8483 let Inst{31-21} = 0b01000100101; 8484 let Inst{20-19} = idx; 8485 let Inst{18-16} = Zm; 8486 let Inst{15-11} = 0b00011; 8487 let Inst{10} = U; 8488 let Inst{9-5} = Zn; 8489 let Inst{4-0} = Zda; 8490 8491 let Constraints = "$Zda = $_Zda"; 8492 let DestructiveInstType = DestructiveOther; 8493 let ElementSize = ZPR32.ElementSize; 8494} 8495 8496multiclass sve_int_dot_mixed_indexed<bit U, string asm, SDPatternOperator op> { 8497 def NAME : sve_int_dot_mixed_indexed<U, asm>; 8498 8499 def : SVE_4_Op_Imm_Pat<nxv4i32, op, nxv4i32, nxv16i8, nxv16i8, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>; 8500} 8501 8502//===----------------------------------------------------------------------===// 8503// SVE Floating Point Matrix Multiply Accumulate Group 8504//===----------------------------------------------------------------------===// 8505 8506class sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty> 8507: I<(outs zprty:$Zda), (ins zprty:$_Zda, zprty:$Zn, zprty:$Zm), 8508 asm, "\t$Zda, $Zn, $Zm", "", []>, Sched<[]> { 8509 bits<5> Zda; 8510 bits<5> Zn; 8511 bits<5> Zm; 8512 let Inst{31-23} = 0b011001001; 8513 let Inst{22} = sz; 8514 let Inst{21} = 1; 8515 let Inst{20-16} = Zm; 8516 let Inst{15-10} = 0b111001; 8517 let Inst{9-5} = Zn; 8518 let Inst{4-0} = Zda; 8519 8520 let Constraints = "$Zda = $_Zda"; 8521 let DestructiveInstType = DestructiveOther; 8522 let ElementSize = zprty.ElementSize; 8523} 8524 8525multiclass sve_fp_matrix_mla<bit sz, string asm, ZPRRegOp zprty, SDPatternOperator op, ValueType vt> { 8526 def NAME : sve_fp_matrix_mla<sz, asm, zprty>; 8527 8528 def : SVE_3_Op_Pat<vt, op , vt, vt, vt, !cast<Instruction>(NAME)>; 8529} 8530 8531//===----------------------------------------------------------------------===// 8532// SVE Memory - Contiguous Load And Replicate 256-bit Group 8533//===----------------------------------------------------------------------===// 8534 8535class sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand VecList> 8536: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 8537 asm, "\t$Zt, $Pg/z, [$Rn, $imm4]", "", []>, Sched<[]> { 8538 bits<5> Zt; 8539 bits<5> Rn; 8540 bits<3> Pg; 8541 bits<4> imm4; 8542 let Inst{31-25} = 0b1010010; 8543 let Inst{24-23} = sz; 8544 let Inst{22-20} = 0b010; 8545 let Inst{19-16} = imm4; 8546 let Inst{15-13} = 0b001; 8547 let Inst{12-10} = Pg; 8548 let Inst{9-5} = Rn; 8549 let Inst{4-0} = Zt; 8550 8551 let mayLoad = 1; 8552} 8553 8554multiclass sve_mem_ldor_si<bits<2> sz, string asm, RegisterOperand listty, 8555 ZPRRegOp zprty, ValueType Ty, ValueType PredTy, SDNode Ld1ro> { 8556 def NAME : sve_mem_ldor_si<sz, asm, listty>; 8557 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8558 (!cast<Instruction>(NAME) listty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 8559 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn]", 8560 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 8561 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $imm4]", 8562 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s32:$imm4), 0>; 8563 8564 // Base addressing mode 8565 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), GPR64sp:$base)), 8566 (!cast<Instruction>(NAME) PPR3bAny:$Pg, GPR64sp:$base, (i64 0))>; 8567 let AddedComplexity = 2 in { 8568 // Reg + Imm addressing mode 8569 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$Pg), (add GPR64:$base, (i64 simm4s32:$imm)))), 8570 (!cast<Instruction>(NAME) $Pg, $base, simm4s32:$imm)>; 8571 } 8572} 8573 8574class sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand VecList, 8575 RegisterOperand gprty> 8576: I<(outs VecList:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 8577 asm, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", []>, Sched<[]> { 8578 bits<5> Zt; 8579 bits<3> Pg; 8580 bits<5> Rn; 8581 bits<5> Rm; 8582 let Inst{31-25} = 0b1010010; 8583 let Inst{24-23} = sz; 8584 let Inst{22-21} = 0b01; 8585 let Inst{20-16} = Rm; 8586 let Inst{15-13} = 0; 8587 let Inst{12-10} = Pg; 8588 let Inst{9-5} = Rn; 8589 let Inst{4-0} = Zt; 8590 8591 let mayLoad = 1; 8592} 8593 8594multiclass sve_mem_ldor_ss<bits<2> sz, string asm, RegisterOperand listty, 8595 ZPRRegOp zprty, RegisterOperand gprty, ValueType Ty, 8596 ValueType PredTy, SDNode Ld1ro, ComplexPattern AddrCP> { 8597 def NAME : sve_mem_ldor_ss<sz, asm, listty, gprty>; 8598 8599 def : InstAlias<asm # "\t$Zt, $Pg/z, [$Rn, $Rm]", 8600 (!cast<Instruction>(NAME) zprty:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprty:$Rm), 0>; 8601 8602 def : Pat<(Ty (Ld1ro (PredTy PPR3bAny:$gp), (AddrCP GPR64sp:$base, gprty:$offset))), 8603 (!cast<Instruction>(NAME) PPR3bAny:$gp, GPR64sp:$base, gprty:$offset)>; 8604} 8605 8606//===----------------------------------------------------------------------===// 8607// SVE Interleave 128-bit Elements Group 8608//===----------------------------------------------------------------------===// 8609 8610class sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm> 8611: I<(outs ZPR128:$Zd), (ins ZPR128:$Zn, ZPR128:$Zm), 8612 asm, "\t$Zd, $Zn, $Zm", 8613 "", 8614 []>, Sched<[]> { 8615 bits<5> Zd; 8616 bits<5> Zm; 8617 bits<5> Zn; 8618 let Inst{31-21} = 0b00000101101; 8619 let Inst{20-16} = Zm; 8620 let Inst{15-13} = 0b000; 8621 let Inst{12-11} = opc; 8622 let Inst{10} = P; 8623 let Inst{9-5} = Zn; 8624 let Inst{4-0} = Zd; 8625} 8626 8627multiclass sve_int_perm_bin_perm_128_zz<bits<2> opc, bit P, string asm, SDPatternOperator op> { 8628 def NAME : sve_int_perm_bin_perm_128_zz<opc, P, asm>; 8629 8630 def : SVE_2_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, !cast<Instruction>(NAME)>; 8631 def : SVE_2_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>; 8632 def : SVE_2_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, !cast<Instruction>(NAME)>; 8633 def : SVE_2_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, !cast<Instruction>(NAME)>; 8634 def : SVE_2_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, !cast<Instruction>(NAME)>; 8635 def : SVE_2_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, !cast<Instruction>(NAME)>; 8636 def : SVE_2_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, !cast<Instruction>(NAME)>; 8637 def : SVE_2_Op_Pat<nxv8bf16, op, nxv8bf16, nxv8bf16, !cast<Instruction>(NAME)>; 8638} 8639 8640/// Addressing modes 8641def am_sve_indexed_s4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-8,7>", [], [SDNPWantRoot]>; 8642def am_sve_indexed_s6 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<-32,31>", [], [SDNPWantRoot]>; 8643 8644def am_sve_regreg_lsl0 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<0>", []>; 8645def am_sve_regreg_lsl1 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<1>", []>; 8646def am_sve_regreg_lsl2 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<2>", []>; 8647def am_sve_regreg_lsl3 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<3>", []>; 8648def am_sve_regreg_lsl4 : ComplexPattern<iPTR, 2, "SelectSVERegRegAddrMode<4>", []>; 8649 8650// Predicated pseudo floating point two operand instructions. 8651multiclass sve_fp_bin_pred_hfd<SDPatternOperator op> { 8652 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8653 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8654 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8655 8656 def : SVE_3_Op_Pat<nxv8f16, op, nxv8i1, nxv8f16, nxv8f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8657 def : SVE_3_Op_Pat<nxv4f16, op, nxv4i1, nxv4f16, nxv4f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8658 def : SVE_3_Op_Pat<nxv2f16, op, nxv2i1, nxv2f16, nxv2f16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8659 def : SVE_3_Op_Pat<nxv4f32, op, nxv4i1, nxv4f32, nxv4f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8660 def : SVE_3_Op_Pat<nxv2f32, op, nxv2i1, nxv2f32, nxv2f32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8661 def : SVE_3_Op_Pat<nxv2f64, op, nxv2i1, nxv2f64, nxv2f64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8662} 8663 8664// Predicated pseudo integer two operand instructions. 8665multiclass sve_int_bin_pred_bhsd<SDPatternOperator op> { 8666 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8667 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8668 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8669 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8670 8671 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8672 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8673 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8674 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8675} 8676 8677// As sve_int_bin_pred but when only i32 and i64 vector types are required. 8678multiclass sve_int_bin_pred_sd<SDPatternOperator op> { 8679 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8680 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8681 8682 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8683 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8684} 8685 8686// Predicated pseudo integer two operand instructions. Second operand is an 8687// immediate specified by imm_[bhsd]. 8688multiclass sve_int_shift_pred_bhsd<SDPatternOperator op, 8689 ComplexPattern imm_b, ComplexPattern imm_h, 8690 ComplexPattern imm_s, ComplexPattern imm_d> { 8691 def _UNDEF_B : PredTwoOpImmPseudo<NAME # _B, ZPR8, Operand<i32>, FalseLanesUndef>; 8692 def _UNDEF_H : PredTwoOpImmPseudo<NAME # _H, ZPR16, Operand<i32>, FalseLanesUndef>; 8693 def _UNDEF_S : PredTwoOpImmPseudo<NAME # _S, ZPR32, Operand<i32>, FalseLanesUndef>; 8694 def _UNDEF_D : PredTwoOpImmPseudo<NAME # _D, ZPR64, Operand<i32>, FalseLanesUndef>; 8695 8696 def : SVE_Shift_DupImm_Pred_Pat<nxv16i8, op, nxv16i1, i32, imm_b, !cast<Instruction>(NAME # _UNDEF_B)>; 8697 def : SVE_Shift_DupImm_Pred_Pat<nxv8i16, op, nxv8i1, i32, imm_h, !cast<Instruction>(NAME # _UNDEF_H)>; 8698 def : SVE_Shift_DupImm_Pred_Pat<nxv4i32, op, nxv4i1, i32, imm_s, !cast<Instruction>(NAME # _UNDEF_S)>; 8699 def : SVE_Shift_DupImm_Pred_Pat<nxv2i64, op, nxv2i1, i64, imm_d, !cast<Instruction>(NAME # _UNDEF_D)>; 8700} 8701 8702multiclass sve_int_bin_pred_all_active_bhsd<SDPatternOperator op> { 8703 def _UNDEF_B : PredTwoOpPseudo<NAME # _B, ZPR8, FalseLanesUndef>; 8704 def _UNDEF_H : PredTwoOpPseudo<NAME # _H, ZPR16, FalseLanesUndef>; 8705 def _UNDEF_S : PredTwoOpPseudo<NAME # _S, ZPR32, FalseLanesUndef>; 8706 def _UNDEF_D : PredTwoOpPseudo<NAME # _D, ZPR64, FalseLanesUndef>; 8707 8708 def : SVE_2_Op_Pred_All_Active_Pt<nxv16i8, op, nxv16i1, nxv16i8, nxv16i8, !cast<Pseudo>(NAME # _UNDEF_B)>; 8709 def : SVE_2_Op_Pred_All_Active_Pt<nxv8i16, op, nxv8i1, nxv8i16, nxv8i16, !cast<Pseudo>(NAME # _UNDEF_H)>; 8710 def : SVE_2_Op_Pred_All_Active_Pt<nxv4i32, op, nxv4i1, nxv4i32, nxv4i32, !cast<Pseudo>(NAME # _UNDEF_S)>; 8711 def : SVE_2_Op_Pred_All_Active_Pt<nxv2i64, op, nxv2i1, nxv2i64, nxv2i64, !cast<Pseudo>(NAME # _UNDEF_D)>; 8712} 8713 8714//===----------------------------------------------------------------------===// 8715// SME2 or SVE2.1 Instructions 8716//===----------------------------------------------------------------------===// 8717 8718class sve2p1_fclamp<string asm, bits<2> sz, ZPRRegOp zpr_ty> 8719 : I<(outs zpr_ty:$Zd), (ins zpr_ty:$_Zd, zpr_ty:$Zn, zpr_ty:$Zm), 8720 asm, "\t$Zd, $Zn, $Zm", "", []>, 8721 Sched<[]> { 8722 bits<5> Zm; 8723 bits<5> Zn; 8724 bits<5> Zd; 8725 let Inst{31-24} = 0b01100100; 8726 let Inst{23-22} = sz; 8727 let Inst{21} = 0b1; 8728 let Inst{20-16} = Zm; 8729 let Inst{15-10} = 0b001001; 8730 let Inst{9-5} = Zn; 8731 let Inst{4-0} = Zd; 8732 8733 let Constraints = "$Zd = $_Zd"; 8734 let DestructiveInstType = DestructiveOther; 8735 let ElementSize = zpr_ty.ElementSize; 8736} 8737 8738multiclass sve2p1_fclamp<string asm, SDPatternOperator op> { 8739 def _H : sve2p1_fclamp<asm, 0b01, ZPR16>; 8740 def _S : sve2p1_fclamp<asm, 0b10, ZPR32>; 8741 def _D : sve2p1_fclamp<asm, 0b11, ZPR64>; 8742 8743 def : SVE_3_Op_Pat<nxv8f16, op, nxv8f16, nxv8f16, nxv8f16, !cast<Instruction>(NAME # _H)>; 8744 def : SVE_3_Op_Pat<nxv4f32, op, nxv4f32, nxv4f32, nxv4f32, !cast<Instruction>(NAME # _S)>; 8745 def : SVE_3_Op_Pat<nxv2f64, op, nxv2f64, nxv2f64, nxv2f64, !cast<Instruction>(NAME # _D)>; 8746} 8747 8748// SVE two-way dot product 8749class sve2p1_two_way_dot_vv<string mnemonic, bit u> 8750 : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR16:$Zm), 8751 mnemonic, "\t$Zda, $Zn, $Zm", 8752 "", []>, Sched<[]> { 8753 bits<5> Zda; 8754 bits<5> Zn; 8755 bits<5> Zm; 8756 let Inst{31-21} = 0b01000100000; 8757 let Inst{20-16} = Zm; 8758 let Inst{15-11} = 0b11001; 8759 let Inst{10} = u; 8760 let Inst{9-5} = Zn; 8761 let Inst{4-0} = Zda; 8762 8763 let Constraints = "$Zda = $_Zda"; 8764 let DestructiveInstType = DestructiveOther; 8765} 8766 8767 8768// SVE two-way dot product (indexed) 8769class sve2p1_two_way_dot_vvi<string mnemonic, bit u> 8770 : I<(outs ZPR32:$Zda), (ins ZPR32:$_Zda, ZPR16:$Zn, ZPR3b16:$Zm, VectorIndexS32b:$i2), 8771 mnemonic, "\t$Zda, $Zn, $Zm$i2", 8772 "", []>, Sched<[]> { 8773 bits<5> Zda; 8774 bits<5> Zn; 8775 bits<3> Zm; 8776 bits<2> i2; 8777 let Inst{31-21} = 0b01000100100; 8778 let Inst{20-19} = i2; 8779 let Inst{18-16} = Zm; 8780 let Inst{15-11} = 0b11001; 8781 let Inst{10} = u; 8782 let Inst{9-5} = Zn; 8783 let Inst{4-0} = Zda; 8784 8785 let Constraints = "$Zda = $_Zda"; 8786 let DestructiveInstType = DestructiveOther; 8787} 8788 8789 8790class sve2p1_ptrue_pn<string mnemonic, bits<2> sz, PNRP8to15RegOp pnrty> 8791 : I<(outs pnrty:$PNd), (ins ), mnemonic, "\t$PNd", 8792 "", []>, Sched<[]> { 8793 bits<3> PNd; 8794 let Inst{31-24} = 0b00100101; 8795 let Inst{23-22} = sz; 8796 let Inst{21-3} = 0b1000000111100000010; 8797 let Inst{2-0} = PNd; 8798} 8799 8800 8801multiclass sve2p1_ptrue_pn<string mnemonic> { 8802 def _B : sve2p1_ptrue_pn<mnemonic, 0b00, PNR8_p8to15>; 8803 def _H : sve2p1_ptrue_pn<mnemonic, 0b01, PNR16_p8to15>; 8804 def _S : sve2p1_ptrue_pn<mnemonic, 0b10, PNR32_p8to15>; 8805 def _D : sve2p1_ptrue_pn<mnemonic, 0b11, PNR64_p8to15>; 8806} 8807 8808 8809// SVE extract mask predicate from predicate-as-counter 8810class sve2p1_pred_as_ctr_to_mask_base<string mnemonic, bits<2> sz, bits<3> opc, 8811 RegisterOperand pprty, Operand idxty> 8812 : I<(outs pprty:$Pd), (ins PNRAny_p8to15:$PNn, idxty:$index), 8813 mnemonic, "\t$Pd, $PNn$index", 8814 "", []>, Sched<[]> { 8815 bits<4> Pd; 8816 bits<3> PNn; 8817 bits<2> imm2; 8818 let Inst{31-24} = 0b00100101; 8819 let Inst{23-22} = sz; 8820 let Inst{21-11} = 0b10000001110; 8821 let Inst{10-8} = opc; 8822 let Inst{7-5} = PNn; 8823 let Inst{4} = 0b1; 8824 let Inst{3-0} = Pd; 8825} 8826 8827class sve2p1_pred_as_ctr_to_mask<string mnemonic, bits<2> sz, PPRRegOp pprty> 8828 : sve2p1_pred_as_ctr_to_mask_base<mnemonic, sz, {0, ?, ?}, pprty, VectorIndexS> { 8829 bits<2> index; 8830 let Inst{9-8} = index; 8831} 8832 8833multiclass sve2p1_pred_as_ctr_to_mask<string mnemonic> { 8834 def _B : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b00, PPR8>; 8835 def _H : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b01, PPR16>; 8836 def _S : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b10, PPR32>; 8837 def _D : sve2p1_pred_as_ctr_to_mask<mnemonic, 0b11, PPR64>; 8838} 8839 8840 8841class sve2p1_pred_as_ctr_to_mask_pair<string mnemonic, bits<2> sz, RegisterOperand pprty> 8842 : sve2p1_pred_as_ctr_to_mask_base<mnemonic, sz, {1, 0, ?}, pprty, VectorIndexD> { 8843 bit index; 8844 let Inst{8} = index; 8845} 8846 8847multiclass sve2p1_pred_as_ctr_to_mask_pair<string mnemonic> { 8848 def _B : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b00, PP_b>; 8849 def _H : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b01, PP_h>; 8850 def _S : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b10, PP_s>; 8851 def _D : sve2p1_pred_as_ctr_to_mask_pair<mnemonic, 0b11, PP_d>; 8852} 8853 8854 8855// SME2 multi-vec extract narrow 8856class sve2p1_multi_vec_extract_narrow<string mnemonic, bits<2> opc, bits<3> tsz> 8857 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn), 8858 mnemonic, "\t$Zd, $Zn", 8859 "", []>, Sched<[]> { 8860 bits<5> Zd; 8861 bits<4> Zn; 8862 let Inst{31-23} = 0b010001010; 8863 let Inst{22} = tsz{2}; 8864 let Inst{21} = 0b1; 8865 let Inst{20-19} = tsz{1-0}; 8866 let Inst{18-13} = 0b001010; 8867 let Inst{12-11} = opc; 8868 let Inst{10} = 0b0; 8869 let Inst{9-6} = Zn; 8870 let Inst{5} = 0b0; 8871 let Inst{4-0} = Zd; 8872} 8873 8874multiclass sve2p1_multi_vec_extract_narrow<string mnemonic, bits<2> opc, SDPatternOperator intrinsic> { 8875 def NAME : sve2p1_multi_vec_extract_narrow<mnemonic, opc, 0b010>; 8876 def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, nxv8i16, nxv4i32>; 8877} 8878 8879// SVE2 multi-vec shift narrow 8880class sve2p1_multi_vec_shift_narrow<string mnemonic, bits<3> opc, bits<2> tsz> 8881 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, vecshiftR16:$imm4), 8882 mnemonic, "\t$Zd, $Zn, $imm4", 8883 "", []>, Sched<[]> { 8884 bits<5> Zd; 8885 bits<4> Zn; 8886 bits<4> imm4; 8887 let Inst{31-23} = 0b010001011; 8888 let Inst{22} = tsz{1}; 8889 let Inst{21} = 0b1; 8890 let Inst{20} = tsz{0}; 8891 let Inst{19-16} = imm4; 8892 let Inst{15-14} = 0b00; 8893 let Inst{13-11} = opc; 8894 let Inst{10} = 0b0; 8895 let Inst{9-6} = Zn; 8896 let Inst{5} = 0b0; 8897 let Inst{4-0} = Zd; 8898} 8899 8900multiclass sve2p1_multi_vec_shift_narrow<string mnemonic, bits<3> opc> { 8901 def : sve2p1_multi_vec_shift_narrow<mnemonic, opc, 0b01>; 8902} 8903 8904 8905// SME2 multi-vec contiguous load (scalar plus scalar, two registers) 8906class sve2p1_mem_cld_ss_2z<string mnemonic, bits<2> msz, bit n, 8907 RegisterOperand vector_ty, RegisterOperand gpr_ty> 8908 : I<(outs vector_ty:$Zt), 8909 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 8910 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 8911 "", []>, Sched<[]> { 8912 bits<4> Zt; 8913 bits<5> Rm; 8914 bits<5> Rn; 8915 bits<3> PNg; 8916 let Inst{31-21} = 0b10100000000; 8917 let Inst{20-16} = Rm; 8918 let Inst{15} = 0b0; 8919 let Inst{14-13} = msz; 8920 let Inst{12-10} = PNg; 8921 let Inst{9-5} = Rn; 8922 let Inst{4-1} = Zt; 8923 let Inst{0} = n; 8924 8925 let mayLoad = 1; 8926} 8927 8928// SME2 multi-vec contiguous load (scalar plus immediate, two registers) 8929class sve2p1_mem_cld_si_2z<string mnemonic, bits<2> msz, bit n, 8930 RegisterOperand vector_ty> 8931 : I<(outs vector_ty:$Zt), 8932 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4), 8933 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]", 8934 "", []>, Sched<[]> { 8935 bits<4> Zt; 8936 bits<5> Rn; 8937 bits<3> PNg; 8938 bits<4> imm4; 8939 let Inst{31-20} = 0b101000000100; 8940 let Inst{19-16} = imm4; 8941 let Inst{15} = 0b0; 8942 let Inst{14-13} = msz; 8943 let Inst{12-10} = PNg; 8944 let Inst{9-5} = Rn; 8945 let Inst{4-1} = Zt; 8946 let Inst{0} = n; 8947 8948 let mayLoad = 1; 8949} 8950 8951multiclass sve2p1_mem_cld_si_2z<string mnemonic, bits<2> msz, bit n, 8952 RegisterOperand vector_ty> { 8953 def NAME : sve2p1_mem_cld_si_2z<mnemonic, msz, n, vector_ty>; 8954 8955 def : InstAlias<mnemonic # " $Zt, $PNg/z, [$Rn]", 8956 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 8957} 8958 8959// SME2 multi-vec contiguous load (scalar plus scalar, four registers) 8960class sve2p1_mem_cld_ss_4z<string mnemonic, bits<2> msz, bit n, 8961 RegisterOperand vector_ty, RegisterOperand gpr_ty> 8962 : I<(outs vector_ty:$Zt), 8963 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 8964 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 8965 "", []>, Sched<[]> { 8966 bits<3> Zt; 8967 bits<5> Rm; 8968 bits<5> Rn; 8969 bits<3> PNg; 8970 let Inst{31-21} = 0b10100000000; 8971 let Inst{20-16} = Rm; 8972 let Inst{15} = 0b1; 8973 let Inst{14-13} = msz; 8974 let Inst{12-10} = PNg; 8975 let Inst{9-5} = Rn; 8976 let Inst{4-2} = Zt; 8977 let Inst{1} = 0b0; 8978 let Inst{0} = n; 8979 8980 let mayLoad = 1; 8981} 8982 8983// SME2 multi-vec contiguous load (scalar plus immediate, four registers) 8984class sve2p1_mem_cld_si_4z<string mnemonic, bits<2> msz, bit n, 8985 RegisterOperand vector_ty> 8986 : I<(outs vector_ty:$Zt), 8987 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4), 8988 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]", 8989 "", []>, Sched<[]> { 8990 bits<3> Zt; 8991 bits<5> Rn; 8992 bits<3> PNg; 8993 bits<4> imm4; 8994 let Inst{31-20} = 0b101000000100; 8995 let Inst{19-16} = imm4; 8996 let Inst{15} = 0b1; 8997 let Inst{14-13} = msz; 8998 let Inst{12-10} = PNg; 8999 let Inst{9-5} = Rn; 9000 let Inst{4-2} = Zt; 9001 let Inst{1} = 0b0; 9002 let Inst{0} = n; 9003 9004 let mayLoad = 1; 9005} 9006 9007multiclass sve2p1_mem_cld_si_4z<string mnemonic, bits<2> msz, bit n, 9008 RegisterOperand vector_ty> { 9009 def NAME : sve2p1_mem_cld_si_4z<mnemonic, msz, n, vector_ty>; 9010 9011 def : InstAlias<mnemonic # " $Zt, $PNg/z, [$Rn]", 9012 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 9013} 9014 9015 9016// SME2 multi-vec contiguous store (scalar plus scalar, two registers) 9017class sve2p1_mem_cst_ss_2z<string mnemonic, bits<2> msz, bit n, 9018 RegisterOperand vector_ty, RegisterOperand gpr_ty> 9019 : I<(outs ), 9020 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 9021 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 9022 "", []>, Sched<[]> { 9023 bits<4> Zt; 9024 bits<5> Rm; 9025 bits<5> Rn; 9026 bits<3> PNg; 9027 let Inst{31-21} = 0b10100000001; 9028 let Inst{20-16} = Rm; 9029 let Inst{15} = 0b0; 9030 let Inst{14-13} = msz; 9031 let Inst{12-10} = PNg; 9032 let Inst{9-5} = Rn; 9033 let Inst{4-1} = Zt; 9034 let Inst{0} = n; 9035 9036 let mayStore = 1; 9037} 9038 9039 9040// SME2 multi-vec contiguous store (scalar plus immediate, two registers) 9041class sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n, 9042 RegisterOperand vector_ty> 9043 : I<(outs ), 9044 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s2:$imm4), 9045 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]", 9046 "", []>, Sched<[]> { 9047 bits<4> Zt; 9048 bits<5> Rn; 9049 bits<3> PNg; 9050 bits<4> imm4; 9051 let Inst{31-20} = 0b101000000110; 9052 let Inst{19-16} = imm4; 9053 let Inst{15} = 0b0; 9054 let Inst{14-13} = msz; 9055 let Inst{12-10} = PNg; 9056 let Inst{9-5} = Rn; 9057 let Inst{4-1} = Zt; 9058 let Inst{0} = n; 9059 9060 let mayStore = 1; 9061} 9062 9063 9064multiclass sve2p1_mem_cst_si_2z<string mnemonic, bits<2> msz, bit n, 9065 RegisterOperand vector_ty> { 9066 def NAME : sve2p1_mem_cst_si_2z<mnemonic, msz, n, vector_ty>; 9067 9068 def : InstAlias<mnemonic # " $Zt, $PNg, [$Rn]", 9069 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 9070} 9071 9072 9073// SME2 multi-vec contiguous store (scalar plus scalar, four registers) 9074class sve2p1_mem_cst_ss_4z<string mnemonic, bits<2> msz, bit n, 9075 RegisterOperand vector_ty, RegisterOperand gpr_ty> 9076 : I<(outs ), 9077 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 9078 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 9079 "", []>, Sched<[]> { 9080 bits<3> Zt; 9081 bits<5> Rm; 9082 bits<5> Rn; 9083 bits<3> PNg; 9084 let Inst{31-21} = 0b10100000001; 9085 let Inst{20-16} = Rm; 9086 let Inst{15} = 0b1; 9087 let Inst{14-13} = msz; 9088 let Inst{12-10} = PNg; 9089 let Inst{9-5} = Rn; 9090 let Inst{4-2} = Zt; 9091 let Inst{1} = 0b0; 9092 let Inst{0} = n; 9093 9094 let mayStore = 1; 9095} 9096 9097 9098// SME2 multi-vec contiguous store (scalar plus immediate, four registers) 9099class sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n, 9100 RegisterOperand vector_ty> 9101 : I<(outs ), 9102 (ins vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, simm4s4:$imm4), 9103 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]", 9104 "", []>, Sched<[]> { 9105 bits<3> Zt; 9106 bits<5> Rn; 9107 bits<3> PNg; 9108 bits<4> imm4; 9109 let Inst{31-20} = 0b101000000110; 9110 let Inst{19-16} = imm4; 9111 let Inst{15} = 0b1; 9112 let Inst{14-13} = msz; 9113 let Inst{12-10} = PNg; 9114 let Inst{9-5} = Rn; 9115 let Inst{4-2} = Zt; 9116 let Inst{1} = 0b0; 9117 let Inst{0} = n; 9118 9119 let mayStore = 1; 9120} 9121 9122 9123multiclass sve2p1_mem_cst_si_4z<string mnemonic, bits<2> msz, bit n, 9124 RegisterOperand vector_ty> { 9125 def NAME : sve2p1_mem_cst_si_4z<mnemonic, msz, n, vector_ty>; 9126 9127 def : InstAlias<mnemonic # " $Zt, $PNg, [$Rn]", 9128 (!cast<Instruction>(NAME) vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 9129} 9130 9131// SVE predicate count (predicate-as-counter) 9132class sve2p1_pcount_pn<string mnemonic, bits<3> opc, bits<2> sz, PNRRegOp pnrty> 9133 : I<(outs GPR64:$Rd), 9134 (ins pnrty:$PNn, sve_vec_len_specifier_enum:$vl), 9135 mnemonic, "\t$Rd, $PNn, $vl", 9136 "", []>, Sched<[]> { 9137 bits<5> Rd; 9138 bits<4> PNn; 9139 bits<1> vl; 9140 let Inst{31-24} = 0b00100101; 9141 let Inst{23-22} = sz; 9142 let Inst{21-19} = 0b100; 9143 let Inst{18-16} = opc; 9144 let Inst{15-11} = 0b10000; 9145 let Inst{10} = vl; 9146 let Inst{9} = 0b1; 9147 let Inst{8-5} = PNn; 9148 let Inst{4-0} = Rd; 9149} 9150 9151multiclass sve2p1_pcount_pn<string mnemonic, bits<3> opc> { 9152 def _B : sve2p1_pcount_pn<mnemonic, opc, 0b00, PNR8>; 9153 def _H : sve2p1_pcount_pn<mnemonic, opc, 0b01, PNR16>; 9154 def _S : sve2p1_pcount_pn<mnemonic, opc, 0b10, PNR32>; 9155 def _D : sve2p1_pcount_pn<mnemonic, opc, 0b11, PNR64>; 9156} 9157 9158 9159// SVE integer compare scalar count and limit (predicate-as-counter) 9160class sve2p1_int_while_rr_pn<string mnemonic, bits<2> sz, bits<3> opc, 9161 PNRP8to15RegOp pnrty> 9162 : I<(outs pnrty:$PNd), (ins GPR64:$Rn, GPR64:$Rm, sve_vec_len_specifier_enum:$vl), 9163 mnemonic, "\t$PNd, $Rn, $Rm, $vl", 9164 "", []>, Sched<[]> { 9165 bits<3> PNd; 9166 bits<5> Rn; 9167 bits<1> vl; 9168 bits<5> Rm; 9169 let Inst{31-24} = 0b00100101; 9170 let Inst{23-22} = sz; 9171 let Inst{21} = 0b1; 9172 let Inst{20-16} = Rm; 9173 let Inst{15-14} = 0b01; 9174 let Inst{13} = vl; 9175 let Inst{12} = 0b0; 9176 let Inst{11-10} = opc{2-1}; 9177 let Inst{9-5} = Rn; 9178 let Inst{4} = 0b1; 9179 let Inst{3} = opc{0}; 9180 let Inst{2-0} = PNd; 9181 9182 let Defs = [NZCV]; 9183} 9184 9185 9186multiclass sve2p1_int_while_rr_pn<string mnemonic, bits<3> opc> { 9187 def _B : sve2p1_int_while_rr_pn<mnemonic, 0b00, opc, PNR8_p8to15>; 9188 def _H : sve2p1_int_while_rr_pn<mnemonic, 0b01, opc, PNR16_p8to15>; 9189 def _S : sve2p1_int_while_rr_pn<mnemonic, 0b10, opc, PNR32_p8to15>; 9190 def _D : sve2p1_int_while_rr_pn<mnemonic, 0b11, opc, PNR64_p8to15>; 9191} 9192 9193 9194// SVE integer compare scalar count and limit (predicate pair) 9195class sve2p1_int_while_rr_pair<string mnemonic, bits<2> sz, bits<3> opc, 9196 RegisterOperand ppr_ty> 9197 : I<(outs ppr_ty:$Pd), (ins GPR64:$Rn, GPR64:$Rm), 9198 mnemonic, "\t$Pd, $Rn, $Rm", 9199 "", []>, Sched<[]> { 9200 bits<3> Pd; 9201 bits<5> Rn; 9202 bits<5> Rm; 9203 let Inst{31-24} = 0b00100101; 9204 let Inst{23-22} = sz; 9205 let Inst{21} = 0b1; 9206 let Inst{20-16} = Rm; 9207 let Inst{15-12} = 0b0101; 9208 let Inst{11-10} = opc{2-1}; 9209 let Inst{9-5} = Rn; 9210 let Inst{4} = 0b1; 9211 let Inst{3-1} = Pd; 9212 let Inst{0} = opc{0}; 9213 9214 let Defs = [NZCV]; 9215} 9216 9217 9218multiclass sve2p1_int_while_rr_pair<string mnemonic, bits<3> opc> { 9219 def _B : sve2p1_int_while_rr_pair<mnemonic, 0b00, opc, PP_b_mul_r>; 9220 def _H : sve2p1_int_while_rr_pair<mnemonic, 0b01, opc, PP_h_mul_r>; 9221 def _S : sve2p1_int_while_rr_pair<mnemonic, 0b10, opc, PP_s_mul_r>; 9222 def _D : sve2p1_int_while_rr_pair<mnemonic, 0b11, opc, PP_d_mul_r>; 9223} 9224 9225 9226class sve_mem_128b_gld_64_unscaled<string mnemonic> 9227 : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 9228 mnemonic, "\t$Zt, $Pg/z, [$Zn, $Rm]", 9229 "", []>, Sched<[]> { 9230 bits<5> Zt; 9231 bits<5> Zn; 9232 bits<3> Pg; 9233 bits<5> Rm; 9234 let Inst{31-21} = 0b11000100000; 9235 let Inst{20-16} = Rm; 9236 let Inst{15-13} = 0b101; 9237 let Inst{12-10} = Pg; 9238 let Inst{9-5} = Zn; 9239 let Inst{4-0} = Zt; 9240 9241 let mayLoad = 1; 9242} 9243 9244 9245multiclass sve_mem_128b_gld_64_unscaled<string mnemonic> { 9246 def NAME : sve_mem_128b_gld_64_unscaled<mnemonic>; 9247 9248 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Zn]", 9249 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 9250} 9251 9252class sve_mem_sst_128b_64_unscaled<string mnemonic> 9253 : I<(outs ), (ins Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, GPR64:$Rm), 9254 mnemonic, "\t$Zt, $Pg, [$Zn, $Rm]", 9255 "", []>, Sched<[]> { 9256 bits<5> Zt; 9257 bits<5> Zn; 9258 bits<3> Pg; 9259 bits<5> Rm; 9260 let Inst{31-21} = 0b11100100001; 9261 let Inst{20-16} = Rm; 9262 let Inst{15-13} = 0b001; 9263 let Inst{12-10} = Pg; 9264 let Inst{9-5} = Zn; 9265 let Inst{4-0} = Zt; 9266 9267 let mayStore = 1; 9268} 9269 9270 9271multiclass sve_mem_sst_128b_64_unscaled<string mnemonic> { 9272 def NAME : sve_mem_sst_128b_64_unscaled<mnemonic>; 9273 9274 def : InstAlias<mnemonic # " $Zt, $Pg, [$Zn]", 9275 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, ZPR64:$Zn, XZR), 1>; 9276} 9277 9278 9279// SVE contiguous load (quadwords, scalar plus immediate) 9280class sve_mem_128b_cld_si<bits<2> dtype, string mnemonic> 9281 : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 9282 mnemonic, "\t$Zt, $Pg/z, [$Rn, $imm4, mul vl]", 9283 "", []>, Sched<[]> { 9284 bits<5> Zt; 9285 bits<5> Rn; 9286 bits<3> Pg; 9287 bits<4> imm4; 9288 let Inst{31-25} = 0b1010010; 9289 let Inst{24-23} = dtype; 9290 let Inst{22-20} = 0b001; 9291 let Inst{19-16} = imm4; 9292 let Inst{15-13} = 0b001; 9293 let Inst{12-10} = Pg; 9294 let Inst{9-5} = Rn; 9295 let Inst{4-0} = Zt; 9296 9297 let mayLoad = 1; 9298} 9299 9300multiclass sve_mem_128b_cld_si<bits<2> dtype, string mnemonic> { 9301 def NAME : sve_mem_128b_cld_si<dtype, mnemonic>; 9302 9303 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]", 9304 (!cast<Instruction>(NAME) Z_q:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 1>; 9305 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn]", 9306 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, 0), 0>; 9307 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $imm4, mul vl]", 9308 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, simm4s1:$imm4), 0>; 9309} 9310 9311 9312// SVE contiguous load (quadwords, scalar plus scalar) 9313class sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty> 9314 : I<(outs Z_q:$Zt), (ins PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm), 9315 mnemonic, "\t$Zt, $Pg/z, [$Rn, $Rm]", "", 9316 []>, Sched<[]> { 9317 bits<5> Zt; 9318 bits<5> Rn; 9319 bits<3> Pg; 9320 bits<5> Rm; 9321 let Inst{31-25} = 0b1010010; 9322 let Inst{24-23} = dtype; 9323 let Inst{22-21} = 0b00; 9324 let Inst{20-16} = Rm; 9325 let Inst{15-13} = 0b100; 9326 let Inst{12-10} = Pg; 9327 let Inst{9-5} = Rn; 9328 let Inst{4-0} = Zt; 9329 9330 let mayLoad = 1; 9331} 9332 9333multiclass sve_mem_128b_cld_ss<bits<2> dtype, string mnemonic, RegisterOperand gprsh_ty> { 9334 def NAME : sve_mem_128b_cld_ss<dtype, mnemonic, gprsh_ty>; 9335 9336 def : InstAlias<mnemonic # " $Zt, $Pg/z, [$Rn, $Rm]", 9337 (!cast<Instruction>(NAME) ZPR128:$Zt, PPR3bAny:$Pg, GPR64sp:$Rn, gprsh_ty:$Rm), 0>; 9338} 9339 9340 9341// SVE floating-point recursive reduction (quadwords) 9342class sve2p1_fp_reduction_q<bits<2> sz, bits<3> opc, string mnemonic, 9343 RegisterOperand zpr_ty, string vec_sfx> 9344 : I<(outs V128:$Vd), (ins PPR3bAny:$Pg, zpr_ty:$Zn), 9345 mnemonic, "\t$Vd." # vec_sfx # ", $Pg, $Zn", 9346 "", []>, Sched<[]> { 9347 bits<5> Vd; 9348 bits<5> Zn; 9349 bits<3> Pg; 9350 let Inst{31-24} = 0b01100100; 9351 let Inst{23-22} = sz; 9352 let Inst{21-19} = 0b010; 9353 let Inst{18-16} = opc; 9354 let Inst{15-13} = 0b101; 9355 let Inst{12-10} = Pg; 9356 let Inst{9-5} = Zn; 9357 let Inst{4-0} = Vd; 9358} 9359 9360multiclass sve2p1_fp_reduction_q<bits<3> opc, string mnemonic> { 9361 def _H : sve2p1_fp_reduction_q<0b01, opc, mnemonic, ZPR16, "8h">; 9362 def _S : sve2p1_fp_reduction_q<0b10, opc, mnemonic, ZPR32, "4s">; 9363 def _D : sve2p1_fp_reduction_q<0b11, opc, mnemonic, ZPR64, "2d">; 9364} 9365 9366 9367// SVE Permute Vector - Quadwords (DUPQ) 9368class sve2p1_dupq<bits<5> ind_tsz, string mnemonic, ZPRRegOp zprty, Operand itype> 9369 : I<(outs zprty:$Zd), (ins zprty:$Zn, itype:$index), 9370 mnemonic, "\t$Zd, $Zn$index", 9371 "", []>, Sched<[]> { 9372 bits<5> Zd; 9373 bits<5> Zn; 9374 let Inst{31-21} = 0b00000101001; 9375 let Inst{20-16} = ind_tsz; 9376 let Inst{15-10} = 0b001001; 9377 let Inst{9-5} = Zn; 9378 let Inst{4-0} = Zd; 9379} 9380 9381multiclass sve2p1_dupq<string mnemonic> { 9382 def _B : sve2p1_dupq<{?, ?, ?, ?, 1}, mnemonic, ZPR8, VectorIndexB32b> { 9383 bits<4> index; 9384 let Inst{20-17} = index; 9385 } 9386 def _H : sve2p1_dupq<{?, ?, ?, 1, 0}, mnemonic, ZPR16, VectorIndexH32b> { 9387 bits<3> index; 9388 let Inst{20-18} = index; 9389 } 9390 def _S : sve2p1_dupq<{?, ?, 1, 0, 0}, mnemonic, ZPR32, VectorIndexS32b> { 9391 bits<2> index; 9392 let Inst{20-19} = index; 9393 } 9394 def _D : sve2p1_dupq<{?, 1, 0, 0, 0}, mnemonic, ZPR64, VectorIndexD32b> { 9395 bits<1> index; 9396 let Inst{20} = index; 9397 } 9398} 9399 9400 9401// SVE Permute Vector - Quadwords (EXTQ) 9402class sve2p1_extq<string mnemonic> 9403 : I<(outs ZPR8:$Zdn), (ins ZPR8:$_Zdn, ZPR8:$Zm, imm0_15:$imm4), 9404 mnemonic, "\t$Zdn, $_Zdn, $Zm, $imm4", 9405 "", []>, Sched<[]> { 9406 bits<5> Zdn; 9407 bits<5> Zm; 9408 bits<4> imm4; 9409 let Inst{31-20} = 0b000001010110; 9410 let Inst{19-16} = imm4; 9411 let Inst{15-10} = 0b001001; 9412 let Inst{9-5} = Zm; 9413 let Inst{4-0} = Zdn; 9414 9415 let Constraints = "$Zdn = $_Zdn"; 9416 let DestructiveInstType = DestructiveOther; 9417 let ElementSize = ZPR8.ElementSize; 9418} 9419 9420 9421// SVE move predicate from vector 9422class sve2p1_vector_to_pred<bits<4> opc, string mnemonic, 9423 PPRRegOp ppr_ty, Operand itype> 9424 : I<(outs ppr_ty:$Pd), (ins ZPRAny:$Zn, itype:$index), 9425 mnemonic, "\t$Pd, $Zn$index", 9426 "", []>, Sched<[]> { 9427 bits<4> Pd; 9428 bits<5> Zn; 9429 let Inst{31-24} = 0b00000101; 9430 let Inst{23-22} = opc{3-2}; 9431 let Inst{21-19} = 0b101; 9432 let Inst{18-17} = opc{1-0}; 9433 let Inst{16-10} = 0b0001110; 9434 let Inst{9-5} = Zn; 9435 let Inst{4} = 0b0; 9436 let Inst{3-0} = Pd; 9437} 9438 9439multiclass sve2p1_vector_to_pred<string mnemonic> { 9440 def _B : sve2p1_vector_to_pred<{0, 0, 0, 1}, mnemonic, PPR8, VectorIndex0>; 9441 def _H : sve2p1_vector_to_pred<{0, 0, 1, ?}, mnemonic, PPR16, VectorIndexD32b> { 9442 bits<1> index; 9443 let Inst{17} = index; 9444 } 9445 def _S : sve2p1_vector_to_pred<{0, 1, ?, ?}, mnemonic, PPR32, VectorIndexS32b> { 9446 bits<2> index; 9447 let Inst{18-17} = index; 9448 } 9449 def _D : sve2p1_vector_to_pred<{1, ?, ?, ?}, mnemonic, PPR64, VectorIndexH32b> { 9450 bits<3> index; 9451 let Inst{22} = index{2}; 9452 let Inst{18-17} = index{1-0}; 9453 } 9454 9455 def : InstAlias<mnemonic # "\t$Pd, $Zn", 9456 (!cast<Instruction>(NAME # _B) PPR8:$Pd, ZPRAny:$Zn, 0), 1>; 9457} 9458 9459 9460// SVE move predicate into vector 9461class sve2p1_pred_to_vector<bits<4> opc, string mnemonic, 9462 PPRRegOp ppr_ty, Operand itype> 9463 : I<(outs ZPRAny:$Zd), (ins ZPRAny:$_Zd, itype:$index, ppr_ty:$Pn), 9464 mnemonic, "\t$Zd$index, $Pn", 9465 "", []>, Sched<[]> { 9466 bits<5> Zd; 9467 bits<4> Pn; 9468 let Inst{31-24} = 0b00000101; 9469 let Inst{23-22} = opc{3-2}; 9470 let Inst{21-19} = 0b101; 9471 let Inst{18-17} = opc{1-0}; 9472 let Inst{16-9} = 0b10011100; 9473 let Inst{8-5} = Pn; 9474 let Inst{4-0} = Zd; 9475 9476 let Constraints = "$Zd = $_Zd"; 9477} 9478 9479multiclass sve2p1_pred_to_vector<string mnemonic> { 9480 def _B : sve2p1_pred_to_vector<{0, 0, 0, 1}, mnemonic, PPR8, VectorIndex0>; 9481 def _H : sve2p1_pred_to_vector<{0, 0, 1, ?}, mnemonic, PPR16, VectorIndexD32b> { 9482 bits<1> index; 9483 let Inst{17} = index; 9484 } 9485 def _S : sve2p1_pred_to_vector<{0, 1, ?, ?}, mnemonic, PPR32, VectorIndexS32b> { 9486 bits<2> index; 9487 let Inst{18-17} = index; 9488 } 9489 def _D : sve2p1_pred_to_vector<{1, ?, ?, ?}, mnemonic, PPR64, VectorIndexH32b> { 9490 bits<3> index; 9491 let Inst{22} = index{2}; 9492 let Inst{18-17} = index{1-0}; 9493 } 9494 9495 def : InstAlias<mnemonic # "\t$Zd, $Pn", 9496 (!cast<Instruction>(NAME # _B) ZPRAny:$Zd, 0, PPR8:$Pn), 1>; 9497} 9498 9499 9500// SVE bitwise logical/add/min/max reductions (quadwords) 9501class sve2p1_int_reduce_q<bits<2> sz, bits<4> opc, string mnemonic, 9502 RegisterOperand zpr_ty, string vec_sfx> 9503 : I<(outs V128:$Vd), (ins PPR3bAny:$Pg, zpr_ty:$Zn), 9504 mnemonic, "\t$Vd." # vec_sfx # ", $Pg, $Zn", 9505 "", []>, Sched<[]> { 9506 bits<5> Vd; 9507 bits<5> Zn; 9508 bits<3> Pg; 9509 let Inst{31-24} = 0b00000100; 9510 let Inst{23-22} = sz; 9511 let Inst{21} = 0b0; 9512 let Inst{20-19} = opc{3-2}; 9513 let Inst{18} = 0b1; 9514 let Inst{17-16} = opc{1-0}; 9515 let Inst{15-13} = 0b001; 9516 let Inst{12-10} = Pg; 9517 let Inst{9-5} = Zn; 9518 let Inst{4-0} = Vd; 9519} 9520 9521multiclass sve2p1_int_reduce_q<bits<4> opc, string mnemonic> { 9522 def _B : sve2p1_int_reduce_q<0b00, opc, mnemonic, ZPR8, "16b">; 9523 def _H : sve2p1_int_reduce_q<0b01, opc, mnemonic, ZPR16, "8h">; 9524 def _S : sve2p1_int_reduce_q<0b10, opc, mnemonic, ZPR32, "4s">; 9525 def _D : sve2p1_int_reduce_q<0b11, opc, mnemonic, ZPR64, "2d">; 9526} 9527 9528 9529// SVE permute vector elements (quadwords) 9530class sve2p1_permute_vec_elems_q<bits<2> sz, bits<3> opc, string mnemonic, 9531 ZPRRegOp zpr_ty, RegisterOperand src1_ty> 9532 : I<(outs zpr_ty:$Zd), (ins src1_ty:$Zn, zpr_ty:$Zm), 9533 mnemonic, "\t$Zd, $Zn, $Zm", 9534 "", []>, Sched<[]> { 9535 bits<5> Zd; 9536 bits<5> Zn; 9537 bits<5> Zm; 9538 let Inst{31-24} = 0b01000100; 9539 let Inst{23-22} = sz; 9540 let Inst{21} = 0b0; 9541 let Inst{20-16} = Zm; 9542 let Inst{15-13} = 0b111; 9543 let Inst{12-10} = opc; 9544 let Inst{9-5} = Zn; 9545 let Inst{4-0} = Zd; 9546} 9547 9548multiclass sve2p1_permute_vec_elems_q<bits<3> opc, string mnemonic> { 9549 def _B : sve2p1_permute_vec_elems_q<0b00, opc, mnemonic, ZPR8, ZPR8>; 9550 def _H : sve2p1_permute_vec_elems_q<0b01, opc, mnemonic, ZPR16, ZPR16>; 9551 def _S : sve2p1_permute_vec_elems_q<0b10, opc, mnemonic, ZPR32, ZPR32>; 9552 def _D : sve2p1_permute_vec_elems_q<0b11, opc, mnemonic, ZPR64, ZPR64>; 9553} 9554 9555multiclass sve2p1_tblq<string mnemonic> { 9556 def _B : sve2p1_permute_vec_elems_q<0b00, 0b110, mnemonic, ZPR8, Z_b>; 9557 def _H : sve2p1_permute_vec_elems_q<0b01, 0b110, mnemonic, ZPR16, Z_h>; 9558 def _S : sve2p1_permute_vec_elems_q<0b10, 0b110, mnemonic, ZPR32, Z_s>; 9559 def _D : sve2p1_permute_vec_elems_q<0b11, 0b110, mnemonic, ZPR64, Z_d>; 9560} 9561