1//=-- SMEInstrFormats.td - AArch64 SME Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Matrix Extension (SME) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def imm_to_tile8 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAB0>", []>; 14def imm_to_tile16 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAH0>", []>; 15def imm_to_tile32 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAS0>", []>; 16def imm_to_tile64 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAD0>", []>; 17def imm_to_tile128 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAQ0>", []>; 18 19def tileslice8 : ComplexPattern<i32 , 2, "SelectSMETileSlice<15, 1>", []>; 20def tileslice16 : ComplexPattern<i32 , 2, "SelectSMETileSlice<7, 1>", []>; 21def tileslice32 : ComplexPattern<i32 , 2, "SelectSMETileSlice<3, 1>", []>; 22def tileslice64 : ComplexPattern<i32 , 2, "SelectSMETileSlice<1, 1>", []>; 23def tileslice128 : ComplexPattern<i32 , 2, "SelectSMETileSlice<0, 1>", []>; // nop 24 25def tileslicerange3s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<14, 2>", []>; 26def tileslicerange2s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<6, 2>", []>; 27 28def am_sme_indexed_b4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0,15>", [], [SDNPWantRoot]>; 29 30//===----------------------------------------------------------------------===// 31// SME Pseudo Classes 32//===----------------------------------------------------------------------===// 33 34def getSMEPseudoMap : InstrMapping { 35 let FilterClass = "SMEPseudo2Instr"; 36 let RowFields = ["PseudoName"]; 37 let ColFields = ["IsInstr"]; 38 let KeyCol = ["0"]; 39 let ValueCols = [["1"]]; 40} 41 42class SMEPseudo2Instr<string name, bit instr> { 43 string PseudoName = name; 44 bit IsInstr = instr; 45} 46 47class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 48 : Pseudo<(outs), (ins i32imm:$tile, PPR3bAny:$pn, PPR3bAny:$pm, 49 zpr_ty:$zn, zpr_ty:$zm), []>, 50 Sched<[]> { 51 // Translated to the actual instructions in AArch64ISelLowering.cpp 52 let SMEMatrixType = za_flag; 53 let usesCustomInserter = 1; 54} 55 56class sme2_za_array_2op_multi_single_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 57 ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 58 : SMEPseudo2Instr<name, 0>, 59 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), []> { 60 let SMEMatrixType = za_flag; 61 let usesCustomInserter = 1; 62} 63 64class sme2_za_array_2op_multi_multi_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 65 SMEMatrixTypeEnum za_flag> 66 : SMEPseudo2Instr<name, 0>, 67 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), []> { 68 let SMEMatrixType = za_flag; 69 let usesCustomInserter = 1; 70} 71 72class sme2_za_array_2op_multi_index_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 73 ZPRRegOp zpr_ty, Operand imm_ty, SMEMatrixTypeEnum za_flag> 74 : SMEPseudo2Instr<name, 0>, 75 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, imm_ty:$i), []> { 76 let SMEMatrixType = za_flag; 77 let usesCustomInserter = 1; 78} 79 80//===----------------------------------------------------------------------===// 81// SME pattern match helpers. 82//===----------------------------------------------------------------------===// 83 84class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 85 ValueType vt, ComplexPattern tileslice> 86 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm), 87 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm)>; 88 89 90class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 91 ValueType vt, ComplexPattern tileslice> 92 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm), 93 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), 94 zpr_ty:$Zm)>; 95class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 96 ValueType vt, ComplexPattern tileslice> 97 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 98 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm), 99 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 100 (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 101 zpr_ty:$Zm)>; 102 103class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> 104 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2), 105 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 106 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), 107 (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>; 108 109class SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> 110 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 111 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm1, vt:$Zm2, vt:$Zm3, vt:$Zm4), 112 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 113 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 114 (REG_SEQUENCE ZPR4Mul4, vt:$Zm1, zsub0, vt:$Zm2, zsub1, vt:$Zm3, zsub2, vt:$Zm4, zsub3))>; 115 116class SME2_ZA_TwoOp_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 117 Operand imm_ty, ComplexPattern tileslice> 118 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm, (i32 imm_ty:$i)), 119 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm, (i32 imm_ty:$i))>; 120 121 122class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 123 Operand imm_ty, ComplexPattern tileslice> 124 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)), 125 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 126 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>; 127 128class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 129 Operand imm_ty, ComplexPattern tileslice> 130 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 131 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)), 132 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 133 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 134 zpr_ty:$Zm, imm_ty:$i)>; 135 136class SME2_Cvt_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt> 137 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4)), 138 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3))>; 139 140//===----------------------------------------------------------------------===// 141// SME Outer Products 142//===----------------------------------------------------------------------===// 143 144class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_ty, 145 ZPRRegOp zpr_ty, string mnemonic> 146 : I<(outs za_ty:$ZAda), 147 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 148 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 149 "", []>, 150 Sched<[]> { 151 bits<5> Zm; 152 bits<3> Pm; 153 bits<3> Pn; 154 bits<5> Zn; 155 let Inst{31-25} = 0b1000000; 156 let Inst{24} = op; 157 let Inst{23} = 0b1; 158 let Inst{22-21} = sz; 159 let Inst{20-16} = Zm; 160 let Inst{15-13} = Pm; 161 let Inst{12-10} = Pn; 162 let Inst{9-5} = Zn; 163 let Inst{4} = S; 164 let Inst{3} = op; 165 166 let Constraints = "$ZAda = $_ZAda"; 167} 168 169multiclass sme_outer_product_fp32<bit S, string mnemonic, SDPatternOperator op> { 170 def NAME : sme_fp_outer_product_inst<S, 0b00, 0b0, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { 171 bits<2> ZAda; 172 let Inst{1-0} = ZAda; 173 let Inst{2} = 0b0; 174 } 175 176 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 177 178 def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm), 179 (nxv4f32 ZPR32:$zn), (nxv4f32 ZPR32:$zm)), 180 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; 181} 182 183multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> { 184 def NAME : sme_fp_outer_product_inst<S, 0b10, 0b0, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { 185 bits<3> ZAda; 186 let Inst{2-0} = ZAda; 187 } 188 189 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 190 191 def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm), 192 (nxv2f64 ZPR64:$zn), (nxv2f64 ZPR64:$zm)), 193 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_7:$tile, $pn, $pm, $zn, $zm)>; 194} 195 196multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s>{ 197 def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b1, TileOp16, ZPR16, mnemonic> { 198 bits<1> ZAda; 199 let Inst{2-1} = 0b00; 200 let Inst{0} = ZAda; 201 } 202} 203 204class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2, 205 MatrixTileOperand za_ty, ZPRRegOp zpr_ty, 206 string mnemonic> 207 : I<(outs za_ty:$ZAda), 208 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 209 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 210 "", []>, 211 Sched<[]> { 212 bits<5> Zm; 213 bits<3> Pm; 214 bits<3> Pn; 215 bits<5> Zn; 216 let Inst{31-25} = 0b1010000; 217 let Inst{24} = opc{2}; // u0 218 let Inst{23} = 0b1; 219 let Inst{22} = sz; 220 let Inst{21} = opc{1}; // u1 221 let Inst{20-16} = Zm; 222 let Inst{15-13} = Pm; 223 let Inst{12-10} = Pn; 224 let Inst{9-5} = Zn; 225 let Inst{4} = opc{0}; //S; 226 let Inst{3} = sme2; 227 228 let Constraints = "$ZAda = $_ZAda"; 229} 230 231multiclass sme_int_outer_product_i32<bits<3> opc, string mnemonic, 232 SDPatternOperator op> { 233 def NAME : sme_int_outer_product_inst<opc, 0b0, 0b0, TileOp32, 234 ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> { 235 bits<2> ZAda; 236 let Inst{1-0} = ZAda; 237 let Inst{2} = 0b0; 238 } 239 240 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 241 242 def : Pat<(op timm32_0_3:$tile, (nxv16i1 PPR3bAny:$pn), (nxv16i1 PPR3bAny:$pm), 243 (nxv16i8 ZPR8:$zn), (nxv16i8 ZPR8:$zm)), 244 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; 245} 246 247multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic, 248 SDPatternOperator op> { 249 def NAME : sme_int_outer_product_inst<opc, 0b1, 0b0, TileOp64, 250 ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> { 251 bits<3> ZAda; 252 let Inst{2-0} = ZAda; 253 } 254 255 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 256 257 def : Pat<(op timm32_0_7:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm), 258 (nxv8i16 ZPR16:$zn), (nxv8i16 ZPR16:$zm)), 259 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_7:$tile, $pn, $pm, $zn, $zm)>; 260} 261 262class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic> 263 : I<(outs TileOp32:$ZAda), 264 (ins TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 265 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 266 "", []>, 267 Sched<[]> { 268 bits<5> Zm; 269 bits<3> Pm; 270 bits<3> Pn; 271 bits<5> Zn; 272 bits<2> ZAda; 273 let Inst{31-25} = 0b1000000; 274 let Inst{24} = !if(opc{2}, 0, 1); 275 let Inst{23-22} = 0b10; 276 let Inst{21} = opc{1}; 277 let Inst{20-16} = Zm; 278 let Inst{15-13} = Pm; 279 let Inst{12-10} = Pn; 280 let Inst{9-5} = Zn; 281 let Inst{4} = opc{0}; 282 let Inst{3} = opc{2}; 283 let Inst{2} = 0b0; 284 let Inst{1-0} = ZAda; 285 286 let Constraints = "$ZAda = $_ZAda"; 287} 288 289multiclass sme_bf16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { 290 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; 291 292 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 293 294 def : Pat<(op timm32_0_3:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm), 295 (nxv8bf16 ZPR16:$zn), (nxv8bf16 ZPR16:$zm)), 296 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; 297} 298 299multiclass sme_f16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { 300 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; 301 302 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 303 304 def : Pat<(op timm32_0_3:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm), 305 (nxv8f16 ZPR16:$zn), (nxv8f16 ZPR16:$zm)), 306 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; 307} 308 309//===----------------------------------------------------------------------===// 310// SME Add Vector to Tile 311//===----------------------------------------------------------------------===// 312 313class sme_add_vector_to_tile_inst<bit op, bit V, MatrixTileOperand tile_ty, 314 ZPRRegOp zpr_ty, string mnemonic> 315 : I<(outs tile_ty:$ZAda), 316 (ins tile_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), 317 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn", 318 "", []>, Sched<[]> { 319 bits<3> Pm; 320 bits<3> Pn; 321 bits<5> Zn; 322 let Inst{31-23} = 0b110000001; 323 let Inst{22} = op; 324 let Inst{21-17} = 0b01000; 325 let Inst{16} = V; 326 let Inst{15-13} = Pm; 327 let Inst{12-10} = Pn; 328 let Inst{9-5} = Zn; 329 let Inst{4-3} = 0b00; 330 331 let Constraints = "$ZAda = $_ZAda"; 332} 333 334class sme_add_vector_to_tile_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 335 : Pseudo<(outs), 336 (ins i32imm:$tile, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), []>, 337 Sched<[]> { 338 // Translated to the actual instructions in AArch64ISelLowering.cpp 339 let SMEMatrixType = za_flag; 340 let usesCustomInserter = 1; 341} 342 343multiclass sme_add_vector_to_tile_u32<bit V, string mnemonic, SDPatternOperator op> { 344 def NAME : sme_add_vector_to_tile_inst<0b0, V, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { 345 bits<2> ZAda; 346 let Inst{2} = 0b0; 347 let Inst{1-0} = ZAda; 348 } 349 350 def _PSEUDO_S : sme_add_vector_to_tile_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 351 352 def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm), 353 (nxv4i32 ZPR32:$zn)), 354 (!cast<Instruction>(NAME # _PSEUDO_S) timm32_0_3:$tile, $pn, $pm, $zn)>; 355} 356 357multiclass sme_add_vector_to_tile_u64<bit V, string mnemonic, SDPatternOperator op> { 358 def NAME : sme_add_vector_to_tile_inst<0b1, V, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { 359 bits<3> ZAda; 360 let Inst{2-0} = ZAda; 361 } 362 363 def _PSEUDO_D : sme_add_vector_to_tile_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 364 365 let Predicates = [HasSMEI16I64] in { 366 def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm), 367 (nxv2i64 ZPR64:$zn)), 368 (!cast<Instruction>(NAME # _PSEUDO_D) timm32_0_7:$tile, $pn, $pm, $zn)>; 369 } 370} 371 372//===----------------------------------------------------------------------===// 373// SME Contiguous Loads 374//===----------------------------------------------------------------------===// 375 376class sme_mem_ld_ss_base<bit Q, bit V, bits<2> msz, dag outs, dag ins, 377 string mnemonic, string argstr> 378 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 379 bits<5> Rm; 380 bits<2> Rv; 381 bits<3> Pg; 382 bits<5> Rn; 383 let Inst{31-25} = 0b1110000; 384 let Inst{24} = Q; 385 let Inst{23-22} = msz; 386 let Inst{21} = 0b0; 387 let Inst{20-16} = Rm; 388 let Inst{15} = V; 389 let Inst{14-13} = Rv; 390 let Inst{12-10} = Pg; 391 let Inst{9-5} = Rn; 392 let Inst{4} = 0b0; 393 394 let mayLoad = 1; 395} 396 397class sme_mem_ld_ss_inst<bit Q, bits<2> msz, string mnemonic, 398 MatrixTileVectorOperand tile_ty, bit is_col, 399 Operand imm_ty, RegisterOperand gpr_ty> 400 : sme_mem_ld_ss_base< 401 Q, is_col, msz, (outs tile_ty:$ZAt), 402 (ins MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, 403 gpr_ty:$Rm), 404 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg/z, [$Rn, $Rm]">; 405 406multiclass sme_mem_ss_aliases_base<string mnemonic, Instruction inst, 407 MatrixTileVectorOperand tile_ty, 408 Operand imm_ty, RegisterOperand gpr_ty, 409 string pg_suffix=""> { 410 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn, $Rm]", 411 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, gpr_ty:$Rm), 0>; 412 // Default XZR offset aliases 413 def : InstAlias<mnemonic # "\t\\{$ZAt[$Rv, $imm]\\}, $Pg" # pg_suffix # ", [$Rn]", 414 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 415 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn]", 416 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 417} 418 419multiclass sme_mem_ss_aliases<string mnemonic, string inst, bit is_col, 420 string pg_suffix=""> { 421 defm : sme_mem_ss_aliases_base<mnemonic # "b", !cast<Instruction>(inst # _B), 422 !if(is_col, TileVectorOpV8, TileVectorOpH8), 423 sme_elm_idx0_15, GPR64shifted8, pg_suffix>; 424 defm : sme_mem_ss_aliases_base<mnemonic # "h", !cast<Instruction>(inst # _H), 425 !if(is_col, TileVectorOpV16, TileVectorOpH16), 426 sme_elm_idx0_7, GPR64shifted16, pg_suffix>; 427 defm : sme_mem_ss_aliases_base<mnemonic # "w", !cast<Instruction>(inst # _S), 428 !if(is_col, TileVectorOpV32, TileVectorOpH32), 429 sme_elm_idx0_3, GPR64shifted32, pg_suffix>; 430 defm : sme_mem_ss_aliases_base<mnemonic # "d", !cast<Instruction>(inst # _D), 431 !if(is_col, TileVectorOpV64, TileVectorOpH64), 432 sme_elm_idx0_1, GPR64shifted64, pg_suffix>; 433 defm : sme_mem_ss_aliases_base<mnemonic # "q", !cast<Instruction>(inst # _Q), 434 !if(is_col, TileVectorOpV128, TileVectorOpH128), 435 sme_elm_idx0_0, GPR64shifted128, pg_suffix>; 436} 437 438multiclass sme_mem_ld_ss_aliases<string inst, bit is_col> { 439 defm NAME : sme_mem_ss_aliases<"ld1", inst, is_col, "/z">; 440} 441 442multiclass sme_mem_ld_ss_patterns<Instruction Inst, SDPatternOperator Load, 443 Operand tile_ty, Operand offset_ty, 444 ComplexPattern addr, 445 ComplexPattern tileslice> { 446 // base, tileslice 447 def : Pat<(Load PPR3bAny:$pg, GPR64sp:$base, tile_ty:$tile, 448 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 449 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, XZR)>; 450 451 // reg + reg, tileslice 452 let AddedComplexity = 1 in { 453 def : Pat<(Load PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), 454 tile_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 455 offset_ty:$imm))), 456 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, $offset)>; 457 } 458} 459 460class sme_load_pseudo 461 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, 462 i32imm:$imm, PPR3bAny:$pg, GPR64sp:$base, GPR64:$offset), []>, 463 Sched<[]> { 464 // Translated to the actual instructions in AArch64ISelLowering.cpp 465 let usesCustomInserter = 1; 466 let mayLoad = 1; 467} 468 469multiclass sme_mem_ld_v_ss<string mnemonic, bit is_col> { 470 def _B : sme_mem_ld_ss_inst<0b0, 0b00, mnemonic # "b", 471 !if(is_col, TileVectorOpV8, TileVectorOpH8), 472 is_col, sme_elm_idx0_15, GPR64shifted8> { 473 bits<4> imm; 474 let Inst{3-0} = imm; 475 } 476 def _H : sme_mem_ld_ss_inst<0b0, 0b01, mnemonic # "h", 477 !if(is_col, TileVectorOpV16, TileVectorOpH16), 478 is_col, sme_elm_idx0_7, GPR64shifted16> { 479 bits<1> ZAt; 480 bits<3> imm; 481 let Inst{3} = ZAt; 482 let Inst{2-0} = imm; 483 } 484 def _S : sme_mem_ld_ss_inst<0b0, 0b10, mnemonic # "w", 485 !if(is_col, TileVectorOpV32, TileVectorOpH32), 486 is_col, sme_elm_idx0_3, GPR64shifted32> { 487 bits<2> ZAt; 488 bits<2> imm; 489 let Inst{3-2} = ZAt; 490 let Inst{1-0} = imm; 491 } 492 def _D : sme_mem_ld_ss_inst<0b0, 0b11, mnemonic # "d", 493 !if(is_col, TileVectorOpV64, TileVectorOpH64), 494 is_col, sme_elm_idx0_1, GPR64shifted64> { 495 bits<3> ZAt; 496 bits<1> imm; 497 let Inst{3-1} = ZAt; 498 let Inst{0} = imm; 499 } 500 def _Q : sme_mem_ld_ss_inst<0b1, 0b11, mnemonic # "q", 501 !if(is_col, TileVectorOpV128, TileVectorOpH128), 502 is_col, sme_elm_idx0_0, GPR64shifted128> { 503 bits<4> ZAt; 504 let Inst{3-0} = ZAt; 505 } 506 507 defm : sme_mem_ld_ss_aliases<NAME, is_col>; 508 509 // Pseudo instructions for lowering intrinsics, using immediates instead of 510 // tile registers. 511 def _PSEUDO_B : sme_load_pseudo; 512 def _PSEUDO_H : sme_load_pseudo; 513 def _PSEUDO_S : sme_load_pseudo; 514 def _PSEUDO_D : sme_load_pseudo; 515 def _PSEUDO_Q : sme_load_pseudo; 516 517 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_B), 518 !if(is_col, int_aarch64_sme_ld1b_vert, 519 int_aarch64_sme_ld1b_horiz), 520 sme_elm_idx0_0, timm32_0_15, am_sve_regreg_lsl0, 521 tileslice8>; 522 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 523 !if(is_col, int_aarch64_sme_ld1h_vert, 524 int_aarch64_sme_ld1h_horiz), 525 timm32_0_1, timm32_0_7, am_sve_regreg_lsl1, 526 tileslice16>; 527 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 528 !if(is_col, int_aarch64_sme_ld1w_vert, 529 int_aarch64_sme_ld1w_horiz), 530 timm32_0_3, timm32_0_3, am_sve_regreg_lsl2, 531 tileslice32>; 532 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 533 !if(is_col, int_aarch64_sme_ld1d_vert, 534 int_aarch64_sme_ld1d_horiz), 535 timm32_0_7, timm32_0_1, am_sve_regreg_lsl3, 536 tileslice64>; 537 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 538 !if(is_col, int_aarch64_sme_ld1q_vert, 539 int_aarch64_sme_ld1q_horiz), 540 timm32_0_15, sme_elm_idx0_0, am_sve_regreg_lsl4, 541 tileslice128>; 542} 543 544multiclass sme_mem_ld_ss<string mnemonic> { 545 defm _H : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b0>; 546 defm _V : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b1>; 547} 548 549//===----------------------------------------------------------------------===// 550// SME Contiguous Stores 551//===----------------------------------------------------------------------===// 552 553class sme_mem_st_ss_base<bit Q, bit V, bits<2> msz, dag ins, 554 string mnemonic, string argstr> 555 : I<(outs), ins, mnemonic, argstr, "", []>, Sched<[]> { 556 bits<5> Rm; 557 bits<2> Rv; 558 bits<3> Pg; 559 bits<5> Rn; 560 let Inst{31-25} = 0b1110000; 561 let Inst{24} = Q; 562 let Inst{23-22} = msz; 563 let Inst{21} = 0b1; 564 let Inst{20-16} = Rm; 565 let Inst{15} = V; 566 let Inst{14-13} = Rv; 567 let Inst{12-10} = Pg; 568 let Inst{9-5} = Rn; 569 let Inst{4} = 0b0; 570 571 let mayStore = 1; 572 let hasSideEffects = 1; 573} 574 575class sme_mem_st_ss_inst<bit Q, bits<2> msz, string mnemonic, 576 MatrixTileVectorOperand tile_ty, bit is_col, 577 Operand imm_ty, RegisterOperand gpr_ty> 578 : sme_mem_st_ss_base< 579 Q, is_col, msz, 580 (ins tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, 581 GPR64sp:$Rn, gpr_ty:$Rm), 582 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg, [$Rn, $Rm]">; 583 584multiclass sme_mem_st_ss_aliases<string inst, bit is_col> { 585 defm NAME : sme_mem_ss_aliases<"st1", inst, is_col>; 586} 587 588multiclass sme_mem_st_ss_patterns<Instruction Inst, SDPatternOperator Store, 589 Operand offset_ty, 590 ComplexPattern imm2tile, 591 ComplexPattern addr, 592 ComplexPattern tileslice> { 593 // base, tileslice 594 def : Pat<(Store PPR3bAny:$pg, GPR64sp:$base, (imm2tile untyped:$tile), 595 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 596 (Inst $tile, $idx, $imm, $pg, $base, XZR)>; 597 598 // reg + reg, tileslice 599 let AddedComplexity = 1 in { 600 def : Pat<(Store PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), 601 (imm2tile untyped:$tile), 602 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 603 (Inst $tile, $idx, $imm, $pg, $base, $offset)>; 604 } 605} 606 607multiclass sme_mem_st_v_ss<string mnemonic, bit is_col> { 608 def _B : sme_mem_st_ss_inst<0b0, 0b00, mnemonic # "b", 609 !if(is_col, TileVectorOpV8, TileVectorOpH8), 610 is_col, sme_elm_idx0_15, GPR64shifted8> { 611 bits<4> imm; 612 let Inst{3-0} = imm; 613 } 614 def _H : sme_mem_st_ss_inst<0b0, 0b01, mnemonic # "h", 615 !if(is_col, TileVectorOpV16, TileVectorOpH16), 616 is_col, sme_elm_idx0_7, GPR64shifted16> { 617 bits<1> ZAt; 618 bits<3> imm; 619 let Inst{3} = ZAt; 620 let Inst{2-0} = imm; 621 } 622 def _S : sme_mem_st_ss_inst<0b0, 0b10, mnemonic # "w", 623 !if(is_col, TileVectorOpV32, TileVectorOpH32), 624 is_col, sme_elm_idx0_3, GPR64shifted32> { 625 bits<2> ZAt; 626 bits<2> imm; 627 let Inst{3-2} = ZAt; 628 let Inst{1-0} = imm; 629 } 630 def _D : sme_mem_st_ss_inst<0b0, 0b11, mnemonic # "d", 631 !if(is_col, TileVectorOpV64, TileVectorOpH64), 632 is_col, sme_elm_idx0_1, GPR64shifted64> { 633 bits<3> ZAt; 634 bits<1> imm; 635 let Inst{3-1} = ZAt; 636 let Inst{0} = imm; 637 } 638 def _Q : sme_mem_st_ss_inst<0b1, 0b11, mnemonic # "q", 639 !if(is_col, TileVectorOpV128, TileVectorOpH128), 640 is_col, sme_elm_idx0_0, GPR64shifted128> { 641 bits<4> ZAt; 642 let Inst{3-0} = ZAt; 643 } 644 645 defm : sme_mem_st_ss_aliases<NAME, is_col>; 646 647 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _B), 648 !if(is_col, int_aarch64_sme_st1b_vert, 649 int_aarch64_sme_st1b_horiz), 650 timm32_0_15, imm_to_tile8, am_sve_regreg_lsl0, 651 tileslice8>; 652 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _H), 653 !if(is_col, int_aarch64_sme_st1h_vert, 654 int_aarch64_sme_st1h_horiz), 655 timm32_0_7, imm_to_tile16, am_sve_regreg_lsl1, 656 tileslice16>; 657 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _S), 658 !if(is_col, int_aarch64_sme_st1w_vert, 659 int_aarch64_sme_st1w_horiz), 660 timm32_0_3, imm_to_tile32, am_sve_regreg_lsl2, 661 tileslice32>; 662 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _D), 663 !if(is_col, int_aarch64_sme_st1d_vert, 664 int_aarch64_sme_st1d_horiz), 665 timm32_0_1, imm_to_tile64, am_sve_regreg_lsl3, 666 tileslice64>; 667 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _Q), 668 !if(is_col, int_aarch64_sme_st1q_vert, 669 int_aarch64_sme_st1q_horiz), 670 sme_elm_idx0_0, imm_to_tile128, 671 am_sve_regreg_lsl4, tileslice128>; 672} 673 674multiclass sme_mem_st_ss<string mnemonic> { 675 defm _H : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b0>; 676 defm _V : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b1>; 677} 678 679//===----------------------------------------------------------------------===// 680// SME Save and Restore Array 681//===----------------------------------------------------------------------===// 682 683class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr> 684 : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "", 685 []>, 686 Sched<[]> { 687 bits<2> Rv; 688 bits<5> Rn; 689 bits<4> imm4; 690 let Inst{31-22} = 0b1110000100; 691 let Inst{21} = isStore; 692 let Inst{20-15} = 0b000000; 693 let Inst{14-13} = Rv; 694 let Inst{12-10} = 0b000; 695 let Inst{9-5} = Rn; 696 let Inst{4} = 0b0; 697 let Inst{3-0} = imm4; 698} 699 700let mayStore = 1 in 701class sme_spill_inst<string opcodestr> 702 : sme_spill_fill_base<0b1, (outs), 703 (ins MatrixOp:$ZAt, MatrixIndexGPR32Op12_15:$Rv, 704 sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 705 imm0_15:$offset), 706 opcodestr>; 707let mayLoad = 1 in 708class sme_fill_inst<string opcodestr> 709 : sme_spill_fill_base<0b0, (outs MatrixOp:$ZAt), 710 (ins MatrixIndexGPR32Op12_15:$Rv, 711 sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 712 imm0_15:$offset), 713 opcodestr>; 714multiclass sme_spill<string opcodestr> { 715 def NAME : sme_spill_inst<opcodestr>; 716 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", 717 (!cast<Instruction>(NAME) MatrixOp:$ZAt, 718 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; 719 // base 720 def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base), 721 (!cast<Instruction>(NAME) ZA, $idx, 0, $base, 0)>; 722 // scalar + immediate (mul vl) 723 let AddedComplexity = 2 in { 724 def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx, 725 (am_sme_indexed_b4 GPR64sp:$base, imm0_15:$imm4)), 726 (!cast<Instruction>(NAME) ZA, $idx, 0, $base, $imm4)>; 727 } 728} 729 730multiclass sme_fill<string opcodestr> { 731 def NAME : sme_fill_inst<opcodestr>; 732 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", 733 (!cast<Instruction>(NAME) MatrixOp:$ZAt, 734 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; 735 def NAME # _PSEUDO 736 : Pseudo<(outs), 737 (ins MatrixIndexGPR32Op12_15:$idx, imm0_15:$imm4, 738 GPR64sp:$base), []>, 739 Sched<[]> { 740 // Translated to actual instruction in AArch64ISelLowering.cpp 741 let usesCustomInserter = 1; 742 let mayLoad = 1; 743 } 744 // base 745 def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base), 746 (!cast<Instruction>(NAME # _PSEUDO) $idx, 0, $base)>; 747 // scalar + immediate (mul vl) 748 let AddedComplexity = 2 in { 749 def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx, 750 (am_sme_indexed_b4 GPR64sp:$base, imm0_15:$imm4)), 751 (!cast<Instruction>(NAME # _PSEUDO) $idx, $imm4, $base)>; 752 } 753} 754 755//===----------------------------------------------------------------------===// 756// Move instructions 757//===----------------------------------------------------------------------===// 758 759class sme_vector_to_tile_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, 760 string mnemonic, string argstr> 761 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 762 bits<2> Rv; 763 bits<3> Pg; 764 bits<5> Zn; 765 let Inst{31-24} = 0b11000000; 766 let Inst{23-22} = sz; 767 let Inst{21-17} = 0b00000; 768 let Inst{16} = Q; 769 let Inst{15} = V; 770 let Inst{14-13} = Rv; 771 let Inst{12-10} = Pg; 772 let Inst{9-5} = Zn; 773 let Inst{4} = 0b0; 774} 775 776class sme_vector_to_tile_inst<bit Q, bits<2> sz, MatrixTileVectorOperand tile_ty, 777 bit is_col, Operand imm_ty, ZPRRegOp zpr_ty, 778 string mnemonic> 779 : sme_vector_to_tile_base<Q, is_col, sz, (outs tile_ty:$ZAd), 780 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 781 mnemonic, "\t$ZAd[$Rv, $imm], $Pg/m, $Zn">{ 782 783 let Constraints = "$ZAd = $_ZAd"; 784} 785 786 787multiclass sme_vector_to_tile_aliases<Instruction inst, 788 MatrixTileVectorOperand tile_ty, 789 ZPRRegOp zpr_ty, Operand imm_ty> { 790 def : InstAlias<"mov\t$ZAd[$Rv, $imm], $Pg/m, $Zn", 791 (inst tile_ty:$ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 1>; 792} 793 794multiclass sme_vector_to_tile_patterns<Instruction inst, ValueType zpr_vt, 795 ValueType ppr_vt, Operand imm_ty, 796 Operand offset_ty, 797 SDPatternOperator op, 798 ComplexPattern tileslice> { 799 def : Pat<(op imm_ty:$tile, MatrixIndexGPR32Op12_15:$idx, 800 (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)), 801 (inst imm_ty:$tile, $idx, 0, $pg, $zn)>; 802 let AddedComplexity = 1 in { 803 def : Pat<(op imm_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 804 offset_ty:$imm)), 805 (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)), 806 (inst imm_ty:$tile, $idx, $imm, $pg, $zn)>; 807 } 808} 809 810class sme_mova_insert_pseudo<SMEMatrixTypeEnum za_flag> 811 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, 812 i32imm:$imm, PPR3bAny:$pg, ZPRAny:$zn), []>, 813 Sched<[]> { 814 // Translated to the actual instructions in AArch64ISelLowering.cpp 815 let SMEMatrixType = za_flag; 816 let usesCustomInserter = 1; 817} 818 819multiclass sme_vector_v_to_tile<string mnemonic, bit is_col> { 820 def _B : sme_vector_to_tile_inst<0b0, 0b00, !if(is_col, TileVectorOpV8, 821 TileVectorOpH8), 822 is_col, sme_elm_idx0_15, ZPR8, mnemonic>, 823 SMEPseudo2Instr<NAME # _B, 1> { 824 bits<4> imm; 825 let Inst{3-0} = imm; 826 } 827 def _H : sme_vector_to_tile_inst<0b0, 0b01, !if(is_col, TileVectorOpV16, 828 TileVectorOpH16), 829 is_col, sme_elm_idx0_7, ZPR16, mnemonic>, 830 SMEPseudo2Instr<NAME # _H, 1> { 831 bits<1> ZAd; 832 bits<3> imm; 833 let Inst{3} = ZAd; 834 let Inst{2-0} = imm; 835 } 836 def _S : sme_vector_to_tile_inst<0b0, 0b10, !if(is_col, TileVectorOpV32, 837 TileVectorOpH32), 838 is_col, sme_elm_idx0_3, ZPR32, mnemonic>, 839 SMEPseudo2Instr<NAME # _S, 1> { 840 bits<2> ZAd; 841 bits<2> imm; 842 let Inst{3-2} = ZAd; 843 let Inst{1-0} = imm; 844 } 845 def _D : sme_vector_to_tile_inst<0b0, 0b11, !if(is_col, TileVectorOpV64, 846 TileVectorOpH64), 847 is_col, sme_elm_idx0_1, ZPR64, mnemonic>, 848 SMEPseudo2Instr<NAME # _D, 1> { 849 bits<3> ZAd; 850 bits<1> imm; 851 let Inst{3-1} = ZAd; 852 let Inst{0} = imm; 853 } 854 def _Q : sme_vector_to_tile_inst<0b1, 0b11, !if(is_col, TileVectorOpV128, 855 TileVectorOpH128), 856 is_col, sme_elm_idx0_0, ZPR128, mnemonic>, 857 SMEPseudo2Instr<NAME # _Q, 1> { 858 bits<4> ZAd; 859 bits<1> imm; 860 let Inst{3-0} = ZAd; 861 } 862 863 // Pseudo instructions for lowering intrinsics, using immediates instead of 864 // tile registers. 865 def _PSEUDO_B : sme_mova_insert_pseudo<SMEMatrixTileB>, SMEPseudo2Instr<NAME # _B, 0>; 866 def _PSEUDO_H : sme_mova_insert_pseudo<SMEMatrixTileH>, SMEPseudo2Instr<NAME # _H, 0>; 867 def _PSEUDO_S : sme_mova_insert_pseudo<SMEMatrixTileS>, SMEPseudo2Instr<NAME # _S, 0>; 868 def _PSEUDO_D : sme_mova_insert_pseudo<SMEMatrixTileD>, SMEPseudo2Instr<NAME # _D, 0>; 869 def _PSEUDO_Q : sme_mova_insert_pseudo<SMEMatrixTileQ>, SMEPseudo2Instr<NAME # _Q, 0>; 870 871 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _B), 872 !if(is_col, TileVectorOpV8, 873 TileVectorOpH8), 874 ZPR8, sme_elm_idx0_15>; 875 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _H), 876 !if(is_col, TileVectorOpV16, 877 TileVectorOpH16), 878 ZPR16, sme_elm_idx0_7>; 879 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _S), 880 !if(is_col, TileVectorOpV32, 881 TileVectorOpH32), 882 ZPR32, sme_elm_idx0_3>; 883 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _D), 884 !if(is_col, TileVectorOpV64, 885 TileVectorOpH64), 886 ZPR64, sme_elm_idx0_1>; 887 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _Q), 888 !if(is_col, TileVectorOpV128, 889 TileVectorOpH128), 890 ZPR128, sme_elm_idx0_0>; 891 892 defvar op = !if(is_col, int_aarch64_sme_write_vert, 893 int_aarch64_sme_write_horiz); 894 895 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_B), 896 nxv16i8, nxv16i1, sme_elm_idx0_0, sme_elm_idx0_15, 897 op, tileslice8>; 898 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 899 nxv8i16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 900 op, tileslice16>; 901 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 902 nxv8f16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 903 op, tileslice16>; 904 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 905 nxv8bf16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 906 op, tileslice16>; 907 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 908 nxv4i32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, 909 op, tileslice32>; 910 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 911 nxv4f32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, 912 op, tileslice32>; 913 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 914 nxv2i64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, 915 op, tileslice64>; 916 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 917 nxv2f64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, 918 op, tileslice64>; 919 920 defvar opq = !if(is_col, int_aarch64_sme_writeq_vert, 921 int_aarch64_sme_writeq_horiz); 922 923 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 924 nxv16i8, nxv16i1, sme_elm_idx0_15, 925 sme_elm_idx0_0, opq, tileslice128>; 926 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 927 nxv8i16, nxv8i1, sme_elm_idx0_15, 928 sme_elm_idx0_0, opq, tileslice128>; 929 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 930 nxv8f16, nxv8i1, sme_elm_idx0_15, 931 sme_elm_idx0_0, opq, tileslice128>; 932 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 933 nxv8bf16, nxv8i1, sme_elm_idx0_15, 934 sme_elm_idx0_0, opq, tileslice128>; 935 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 936 nxv4i32, nxv4i1, sme_elm_idx0_15, 937 sme_elm_idx0_0, opq, tileslice128>; 938 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 939 nxv4f32, nxv4i1, sme_elm_idx0_15, 940 sme_elm_idx0_0, opq, tileslice128>; 941 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 942 nxv2i64, nxv2i1, sme_elm_idx0_15, 943 sme_elm_idx0_0, opq, tileslice128>; 944 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 945 nxv2f64, nxv2i1, sme_elm_idx0_15, 946 sme_elm_idx0_0, opq, tileslice128>; 947} 948 949multiclass sme_vector_to_tile<string mnemonic> { 950 defm _H : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b0>; 951 defm _V : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b1>; 952} 953 954class sme_tile_to_vector_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, 955 string mnemonic, string argstr> 956 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 957 bits<2> Rv; 958 bits<3> Pg; 959 bits<5> Zd; 960 let Inst{31-24} = 0b11000000; 961 let Inst{23-22} = sz; 962 let Inst{21-17} = 0b00001; 963 let Inst{16} = Q; 964 let Inst{15} = V; 965 let Inst{14-13} = Rv; 966 let Inst{12-10} = Pg; 967 let Inst{9} = 0b0; 968 let Inst{4-0} = Zd; 969} 970 971class sme_tile_to_vector_inst<bit Q, bits<2> sz, ZPRRegOp zpr_ty, 972 MatrixTileVectorOperand tile_ty, 973 bit is_col, Operand imm_ty, string mnemonic> 974 : sme_tile_to_vector_base<Q, is_col, sz, (outs zpr_ty:$Zd), 975 (ins zpr_ty:$_Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 976 mnemonic, "\t$Zd, $Pg/m, $ZAn[$Rv, $imm]"> { 977 978 let Constraints = "$Zd = $_Zd"; 979} 980 981multiclass sme_tile_to_vector_aliases<Instruction inst, ZPRRegOp zpr_ty, 982 MatrixTileVectorOperand tile_ty, 983 Operand imm_ty > { 984 def : InstAlias<"mov\t$Zd, $Pg/m, $ZAn[$Rv, $imm]", 985 (inst zpr_ty:$Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1>; 986} 987 988multiclass sme_tile_to_vector_patterns<Instruction inst, ValueType zpr_vt, 989 ValueType ppr_vt, Operand offset_ty, 990 ComplexPattern imm2tile, 991 ComplexPattern tileslice, 992 SDPatternOperator op> { 993 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), 994 (imm2tile untyped:$tile), MatrixIndexGPR32Op12_15:$idx)), 995 (inst $passthru, $pg, $tile, $idx, 0)>; 996 let AddedComplexity = 1 in { 997 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), 998 (imm2tile untyped:$tile), 999 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 1000 offset_ty:$imm)))), 1001 (inst $passthru, $pg, $tile, $idx, $imm)>; 1002 } 1003} 1004 1005multiclass sme_tile_to_vector_v<string mnemonic, bit is_col> { 1006 def _B : sme_tile_to_vector_inst<0b0, 0b00, ZPR8, !if(is_col, TileVectorOpV8, 1007 TileVectorOpH8), 1008 is_col, sme_elm_idx0_15, mnemonic> { 1009 bits<4> imm; 1010 let Inst{8-5} = imm; 1011 } 1012 def _H : sme_tile_to_vector_inst<0b0, 0b01, ZPR16, !if(is_col, TileVectorOpV16, 1013 TileVectorOpH16), 1014 is_col, sme_elm_idx0_7, mnemonic> { 1015 bits<1> ZAn; 1016 bits<3> imm; 1017 let Inst{8} = ZAn; 1018 let Inst{7-5} = imm; 1019 } 1020 def _S : sme_tile_to_vector_inst<0b0, 0b10, ZPR32, !if(is_col, TileVectorOpV32, 1021 TileVectorOpH32), 1022 is_col, sme_elm_idx0_3, mnemonic> { 1023 bits<2> ZAn; 1024 bits<2> imm; 1025 let Inst{8-7} = ZAn; 1026 let Inst{6-5} = imm; 1027 } 1028 def _D : sme_tile_to_vector_inst<0b0, 0b11, ZPR64, !if(is_col, TileVectorOpV64, 1029 TileVectorOpH64), 1030 is_col, sme_elm_idx0_1, mnemonic> { 1031 bits<3> ZAn; 1032 bits<1> imm; 1033 let Inst{8-6} = ZAn; 1034 let Inst{5} = imm; 1035 } 1036 def _Q : sme_tile_to_vector_inst<0b1, 0b11, ZPR128, !if(is_col, TileVectorOpV128, 1037 TileVectorOpH128), 1038 is_col, sme_elm_idx0_0, mnemonic> { 1039 bits<4> ZAn; 1040 let Inst{8-5} = ZAn; 1041 } 1042 1043 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _B), ZPR8, 1044 !if(is_col, TileVectorOpV8, 1045 TileVectorOpH8), sme_elm_idx0_15>; 1046 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _H), ZPR16, 1047 !if(is_col, TileVectorOpV16, 1048 TileVectorOpH16), sme_elm_idx0_7>; 1049 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _S), ZPR32, 1050 !if(is_col, TileVectorOpV32, 1051 TileVectorOpH32), sme_elm_idx0_3>; 1052 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _D), ZPR64, 1053 !if(is_col, TileVectorOpV64, 1054 TileVectorOpH64), sme_elm_idx0_1>; 1055 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _Q), ZPR128, 1056 !if(is_col, TileVectorOpV128, 1057 TileVectorOpH128), sme_elm_idx0_0>; 1058 1059 defvar op = !if(is_col, int_aarch64_sme_read_vert, 1060 int_aarch64_sme_read_horiz); 1061 1062 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _B), 1063 nxv16i8, nxv16i1, sme_elm_idx0_15, 1064 imm_to_tile8, tileslice8, op>; 1065 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1066 nxv8i16, nxv8i1, sme_elm_idx0_7, 1067 imm_to_tile16, tileslice16, op>; 1068 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1069 nxv8f16, nxv8i1, sme_elm_idx0_7, 1070 imm_to_tile16, tileslice16, op>; 1071 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1072 nxv8bf16, nxv8i1, sme_elm_idx0_7, 1073 imm_to_tile16, tileslice16, op>; 1074 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), 1075 nxv4i32, nxv4i1, sme_elm_idx0_3, 1076 imm_to_tile32, tileslice32, op>; 1077 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), 1078 nxv4f32, nxv4i1, sme_elm_idx0_3, 1079 imm_to_tile32, tileslice32, op>; 1080 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), 1081 nxv2i64, nxv2i1, sme_elm_idx0_1, 1082 imm_to_tile64, tileslice64, op>; 1083 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), 1084 nxv2f64, nxv2i1, sme_elm_idx0_1, 1085 imm_to_tile64, tileslice64, op>; 1086 1087 defvar opq = !if(is_col, int_aarch64_sme_readq_vert, 1088 int_aarch64_sme_readq_horiz); 1089 1090 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1091 nxv16i8, nxv16i1, sme_elm_idx0_0, 1092 imm_to_tile128, tileslice128, opq>; 1093 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1094 nxv8i16, nxv8i1, sme_elm_idx0_0, 1095 imm_to_tile128, tileslice128, opq>; 1096 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1097 nxv8f16, nxv8i1, sme_elm_idx0_0, 1098 imm_to_tile128, tileslice128, opq>; 1099 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1100 nxv8bf16, nxv8i1, sme_elm_idx0_0, 1101 imm_to_tile128, tileslice128, opq>; 1102 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1103 nxv4i32, nxv4i1, sme_elm_idx0_0, 1104 imm_to_tile128, tileslice128, opq>; 1105 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1106 nxv4f32, nxv4i1, sme_elm_idx0_0, 1107 imm_to_tile128, tileslice128, opq>; 1108 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1109 nxv2i64, nxv2i1, sme_elm_idx0_0, 1110 imm_to_tile128, tileslice128, opq>; 1111 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1112 nxv2f64, nxv2i1, sme_elm_idx0_0, 1113 imm_to_tile128, tileslice128, opq>; 1114} 1115 1116multiclass sme_tile_to_vector<string mnemonic> { 1117 defm _H : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b0>; 1118 defm _V : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b1>; 1119} 1120 1121//===----------------------------------------------------------------------===// 1122// SME Zero 1123//===----------------------------------------------------------------------===// 1124 1125// NOTE: This definition isn't really correct because there are outputs, i.e. 1126// the tile registers being zeroed. We fix this up in a custom inserter that 1127// marks the appropriate registers as being implicitly defined. 1128class sme_zero_inst<string mnemonic> 1129 : I<(outs), (ins MatrixTileList:$imm), 1130 mnemonic, "\t$imm", "", []>, Sched<[]> { 1131 bits<8> imm; 1132 let Inst{31-8} = 0b110000000000100000000000; 1133 let Inst{7-0} = imm; 1134} 1135 1136multiclass sme_zero<string mnemonic> { 1137 def NAME : sme_zero_inst<mnemonic>; 1138 1139 def : InstAlias<"zero\t\\{za\\}", (!cast<Instruction>(NAME) 0b11111111), 1>; 1140 def : InstAlias<"zero\t\\{za0.h\\}", (!cast<Instruction>(NAME) 0b01010101), 1>; 1141 def : InstAlias<"zero\t\\{za1.h\\}", (!cast<Instruction>(NAME) 0b10101010), 1>; 1142 def : InstAlias<"zero\t\\{za0.s\\}", (!cast<Instruction>(NAME) 0b00010001), 1>; 1143 def : InstAlias<"zero\t\\{za1.s\\}", (!cast<Instruction>(NAME) 0b00100010), 1>; 1144 def : InstAlias<"zero\t\\{za2.s\\}", (!cast<Instruction>(NAME) 0b01000100), 1>; 1145 def : InstAlias<"zero\t\\{za3.s\\}", (!cast<Instruction>(NAME) 0b10001000), 1>; 1146 def : InstAlias<"zero\t\\{za0.s,za1.s\\}", (!cast<Instruction>(NAME) 0b00110011), 1>; 1147 def : InstAlias<"zero\t\\{za0.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10011001), 1>; 1148 def : InstAlias<"zero\t\\{za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01100110), 1>; 1149 def : InstAlias<"zero\t\\{za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11001100), 1>; 1150 def : InstAlias<"zero\t\\{za0.s,za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01110111), 1>; 1151 def : InstAlias<"zero\t\\{za0.s,za1.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10111011), 1>; 1152 def : InstAlias<"zero\t\\{za0.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11011101), 1>; 1153 def : InstAlias<"zero\t\\{za1.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11101110), 1>; 1154 1155 def NAME # _PSEUDO : Pseudo<(outs), (ins i32imm:$tilelist), []>, 1156 Sched<[]> { 1157 // Translated to the actual instructions in AArch64ISelLowering.cpp 1158 let usesCustomInserter = 1; 1159 } 1160 1161 def : Pat<(int_aarch64_sme_zero timm32_0_255:$imm), 1162 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_255:$imm)>; 1163} 1164 1165//===----------------------------------------------------------------------===// 1166// SVE2 Instructions 1167//===----------------------------------------------------------------------===// 1168 1169class sve2_int_perm_revd<string asm> 1170 : I<(outs ZPR128:$Zd), (ins ZPR128:$_Zd, PPR3bAny:$Pg, ZPR128:$Zn), 1171 asm, "\t$Zd, $Pg/m, $Zn", "", []>, 1172 Sched<[]> { 1173 bits<5> Zd; 1174 bits<3> Pg; 1175 bits<5> Zn; 1176 let Inst{31-24} = 0b00000101; 1177 let Inst{23-22} = 0b00; // size 1178 let Inst{21-13} = 0b101110100; 1179 let Inst{12-10} = Pg; 1180 let Inst{9-5} = Zn; 1181 let Inst{4-0} = Zd; 1182 1183 let Constraints = "$Zd = $_Zd"; 1184 let DestructiveInstType = DestructiveUnary; 1185 let ElementSize = ZPR128.ElementSize; 1186} 1187 1188multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> { 1189 def NAME : sve2_int_perm_revd<asm>; 1190 1191 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>; 1192 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>; 1193 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>; 1194 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>; 1195} 1196 1197class sve2_clamp<string asm, bits<2> sz, bit U, ZPRRegOp zpr_ty> 1198 : I<(outs zpr_ty:$Zd), (ins zpr_ty:$Zn, zpr_ty:$Zm, zpr_ty:$_Zd), 1199 asm, "\t$Zd, $Zn, $Zm", "", []>, 1200 Sched<[]> { 1201 bits<5> Zm; 1202 bits<5> Zn; 1203 bits<5> Zd; 1204 let Inst{31-24} = 0b01000100; 1205 let Inst{23-22} = sz; 1206 let Inst{21} = 0b0; 1207 let Inst{20-16} = Zm; 1208 let Inst{15-11} = 0b11000; 1209 let Inst{10} = U; 1210 let Inst{9-5} = Zn; 1211 let Inst{4-0} = Zd; 1212 1213 let Constraints = "$Zd = $_Zd"; 1214 let DestructiveInstType = DestructiveOther; 1215 let ElementSize = zpr_ty.ElementSize; 1216} 1217 1218multiclass sve2_clamp<string asm, bit U, SDPatternOperator op> { 1219 def _B : sve2_clamp<asm, 0b00, U, ZPR8>; 1220 def _H : sve2_clamp<asm, 0b01, U, ZPR16>; 1221 def _S : sve2_clamp<asm, 0b10, U, ZPR32>; 1222 def _D : sve2_clamp<asm, 0b11, U, ZPR64>; 1223 1224 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1225 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1226 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1227 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1228} 1229 1230class sve2_int_perm_sel_p<string asm, PPRRegOp ppr_ty, Operand imm_ty> 1231 : I<(outs PPRAny:$Pd), (ins PPRAny:$Pn, ppr_ty:$Pm, 1232 MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1233 asm, "\t$Pd, $Pn, $Pm[$Rv, $imm]", "", []>, 1234 Sched<[]> { 1235 bits<2> Rv; 1236 bits<4> Pn; 1237 bits<4> Pm; 1238 bits<4> Pd; 1239 let Inst{31-24} = 0b00100101; 1240 let Inst{21} = 0b1; 1241 let Inst{17-16} = Rv; 1242 let Inst{15-14} = 0b01; 1243 let Inst{13-10} = Pn; 1244 let Inst{9} = 0b0; 1245 let Inst{8-5} = Pm; 1246 let Inst{4} = 0b0; 1247 let Inst{3-0} = Pd; 1248} 1249 1250multiclass sve2_int_perm_sel_p<string asm, SDPatternOperator op> { 1251 def _B : sve2_int_perm_sel_p<asm, PPR8, sme_elm_idx0_15> { 1252 bits<4> imm; 1253 let Inst{23-22} = imm{3-2}; 1254 let Inst{20-19} = imm{1-0}; 1255 let Inst{18} = 0b1; 1256 } 1257 def _H : sve2_int_perm_sel_p<asm, PPR16, sme_elm_idx0_7> { 1258 bits<3> imm; 1259 let Inst{23-22} = imm{2-1}; 1260 let Inst{20} = imm{0}; 1261 let Inst{19-18} = 0b10; 1262 } 1263 def _S : sve2_int_perm_sel_p<asm, PPR32, sme_elm_idx0_3> { 1264 bits<2> imm; 1265 let Inst{23-22} = imm{1-0}; 1266 let Inst{20-18} = 0b100; 1267 } 1268 def _D : sve2_int_perm_sel_p<asm, PPR64, sme_elm_idx0_1> { 1269 bits<1> imm; 1270 let Inst{23} = imm; 1271 let Inst{22} = 0b1; 1272 let Inst{20-18} = 0b000; 1273 } 1274 1275 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1276 (!cast<Instruction>(NAME # _B) PNRAny:$Pd, 1277 PNRAny:$Pn, PPR8:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm), 0>; 1278 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1279 (!cast<Instruction>(NAME # _H) PNRAny:$Pd, 1280 PNRAny:$Pn, PPR16:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_7:$imm), 0>; 1281 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1282 (!cast<Instruction>(NAME # _S) PNRAny:$Pd, 1283 PNRAny:$Pn, PPR32:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_3:$imm), 0>; 1284 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1285 (!cast<Instruction>(NAME # _D) PNRAny:$Pd, 1286 PNRAny:$Pn, PPR64:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_1:$imm), 0>; 1287 1288 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPRAny:$Pm), 1289 MatrixIndexGPR32Op12_15:$idx)), 1290 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, 0)>; 1291 def : Pat<(nxv8i1 (op (nxv8i1 PPRAny:$Pn), (nxv8i1 PPRAny:$Pm), 1292 MatrixIndexGPR32Op12_15:$idx)), 1293 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, 0)>; 1294 def : Pat<(nxv4i1 (op (nxv4i1 PPRAny:$Pn), (nxv4i1 PPRAny:$Pm), 1295 MatrixIndexGPR32Op12_15:$idx)), 1296 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, 0)>; 1297 def : Pat<(nxv2i1 (op (nxv2i1 PPRAny:$Pn), (nxv2i1 PPRAny:$Pm), 1298 MatrixIndexGPR32Op12_15:$idx)), 1299 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, 0)>; 1300 1301 let AddedComplexity = 1 in { 1302 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPRAny:$Pm), 1303 (i32 (tileslice8 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm)))), 1304 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, $imm)>; 1305 def : Pat<(nxv8i1 (op (nxv8i1 PPRAny:$Pn), (nxv8i1 PPRAny:$Pm), 1306 (i32 (tileslice16 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_7:$imm)))), 1307 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, $imm)>; 1308 def : Pat<(nxv4i1 (op (nxv4i1 PPRAny:$Pn), (nxv4i1 PPRAny:$Pm), 1309 (i32 (tileslice32 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_3:$imm)))), 1310 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, $imm)>; 1311 def : Pat<(nxv2i1 (op (nxv2i1 PPRAny:$Pn), (nxv2i1 PPRAny:$Pm), 1312 (i32 (tileslice64 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_1:$imm)))), 1313 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, $imm)>; 1314 } 1315} 1316 1317//===----------------------------------------------------------------------===// 1318// SME2 Instructions 1319//===----------------------------------------------------------------------===// 1320 1321//===----------------------------------------------------------------------===// 1322// SME2 single-multi ternary int/fp, two/four registers 1323 1324class sme2_dot_mla_add_sub_array_vg24_single<bits<7> op, 1325 MatrixOperand matrix_ty, 1326 RegisterOperand multi_vector_ty, 1327 ZPRRegOp zpr_ty, 1328 string mnemonic> 1329 : I<(outs matrix_ty:$ZAd), 1330 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1331 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 1332 mnemonic,"\t$ZAd[$Rv, $imm3, " # !if(op{5}, "vgx4", "vgx2") # "], $Zn, $Zm", 1333 "", []> , Sched<[]> { 1334 bits<4> Zm; 1335 bits<5> Zn; 1336 bits<2> Rv; 1337 bits<3> imm3; 1338 let Inst{31-23} = 0b110000010; 1339 let Inst{22} = op{6}; //sz 1340 let Inst{21} = 0b1; 1341 let Inst{20} = op{5}; //vgx4 1342 let Inst{19-16} = Zm; 1343 let Inst{15} = 0b0; 1344 let Inst{14-13} = Rv; 1345 let Inst{12-10} = op{4-2}; 1346 let Inst{9-5} = Zn; 1347 let Inst{4-3} = op{1-0}; 1348 let Inst{2-0} = imm3; 1349 let Constraints = "$ZAd = $_ZAd"; 1350} 1351 1352multiclass sme2_dot_mla_add_sub_array_vg24_single<string mnemonic, bits<7> op, 1353 MatrixOperand matrix_ty, 1354 RegisterOperand multi_vector_ty, 1355 ZPRRegOp zpr_ty>{ 1356 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1357 1358 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1359 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1360} 1361 1362multiclass sme2_dot_mla_add_sub_array_vg2_single<string mnemonic, bits<7> op, 1363 MatrixOperand matrix_ty, 1364 RegisterOperand multi_vector_ty, 1365 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ 1366 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1367 1368 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1369 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1370 1371 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; 1372 1373 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; 1374} 1375 1376multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op, 1377 MatrixOperand matrix_ty, 1378 RegisterOperand multi_vector_ty, 1379 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ 1380 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1381 1382 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1383 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1384 1385 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; 1386 1387 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; 1388} 1389 1390//===----------------------------------------------------------------------===// 1391// SME2 multiple vectors ternary INT/FP two and four registers 1392class sme2_dot_mla_add_sub_array_vg2_multi<bits<6> op, 1393 MatrixOperand matrix_ty, 1394 RegisterOperand multi_vector_ty, 1395 string mnemonic> 1396 : I<(outs matrix_ty:$ZAd), 1397 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1398 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 1399 mnemonic, "\t$ZAd[$Rv, $imm3, vgx2], $Zn, $Zm", 1400 "", []>, Sched<[]>{ 1401 bits<4> Zm; 1402 bits<4> Zn; 1403 bits<2> Rv; 1404 bits<3> imm3; 1405 let Inst{31-23} = 0b110000011; 1406 let Inst{22} = op{5}; //sz 1407 let Inst{21} = 0b1; 1408 let Inst{20-17} = Zm; 1409 let Inst{16-15} = 0b00; 1410 let Inst{14-13} = Rv; 1411 let Inst{12-10} = op{4-2}; 1412 let Inst{9-6} = Zn; 1413 let Inst{5} = 0b0; 1414 let Inst{4-3} = op{1-0}; 1415 let Inst{2-0} = imm3; 1416 let Constraints = "$ZAd = $_ZAd"; 1417} 1418 1419multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<6> op, 1420 MatrixOperand matrix_ty, 1421 RegisterOperand multi_vector_ty, ValueType zpr_ty, 1422 SDPatternOperator intrinsic> { 1423 def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1424 1425 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; 1426 1427 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; 1428 1429 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1430 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 1431} 1432 1433class sme2_dot_mla_add_sub_array_vg4_multi<bits<6> op, 1434 MatrixOperand matrix_ty, 1435 RegisterOperand multi_vector_ty, 1436 string mnemonic> 1437 : I<(outs matrix_ty:$ZAd), 1438 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1439 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 1440 mnemonic, "\t$ZAd[$Rv, $imm3, vgx4], $Zn, $Zm", 1441 "", []>, Sched<[]>{ 1442 bits<3> Zm; 1443 bits<3> Zn; 1444 bits<2> Rv; 1445 bits<3> imm3; 1446 let Inst{31-23} = 0b110000011; 1447 let Inst{22} = op{5}; //sz 1448 let Inst{21} = 0b1; 1449 let Inst{20-18} = Zm; 1450 let Inst{17-15} = 0b010; 1451 let Inst{14-13} = Rv; 1452 let Inst{12-10} = op{4-2}; 1453 let Inst{9-7} = Zn; 1454 let Inst{6-5} = 0b00; 1455 let Inst{4-3} = op{1-0}; 1456 let Inst{2-0} = imm3; 1457 let Constraints = "$ZAd = $_ZAd"; 1458} 1459 1460multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<6> op, 1461 MatrixOperand matrix_ty, 1462 RegisterOperand multi_vector_ty, 1463 ValueType zpr_ty, SDPatternOperator intrinsic>{ 1464 def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1465 1466 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; 1467 1468 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; 1469 1470 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1471 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 1472} 1473 1474//===----------------------------------------------------------------------===// 1475// SME2 multiple vectors binary two or four registers 1476 1477class sme2_multivec_accum_add_sub<string mnemonic, bit sz, bit vg4, bits<3> op, 1478 MatrixOperand matrix_ty, 1479 RegisterOperand vector_ty> 1480 : I<(outs matrix_ty:$ZAdn), 1481 (ins matrix_ty:$_ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 1482 mnemonic, "\t$ZAdn[$Rv, $imm3, " # !if(vg4, "vgx4", "vgx2") # "], $Zm", 1483 "", []>, Sched<[]> { 1484 bits<2> Rv; 1485 bits<3> imm3; 1486 let Inst{31-23} = 0b110000011; 1487 let Inst{22} = sz; 1488 let Inst{21-19} = 0b100; 1489 let Inst{18} = op{2}; 1490 let Inst{17} = 0b0; 1491 let Inst{16} = vg4; 1492 let Inst{15} = 0b0; 1493 let Inst{14-13} = Rv; 1494 let Inst{12-10} = 0b111; 1495 let Inst{5} = 0b0; 1496 let Inst{4-3} = op{1-0}; 1497 let Inst{2-0} = imm3; 1498 1499 let Constraints = "$ZAdn = $_ZAdn"; 1500} 1501 1502class sme2_multivec_accum_add_sub_vg2<string mnemonic, bit sz, bits<3> op, 1503 MatrixOperand matrix_ty, 1504 RegisterOperand vector_ty> 1505 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b0, op, matrix_ty, vector_ty> { 1506 bits<4> Zm; 1507 let Inst{9-6} = Zm; 1508} 1509 1510 1511multiclass sme2_multivec_accum_add_sub_vg2<string mnemonic, bits<4> op, 1512 MatrixOperand matrix_ty, 1513 RegisterOperand vector_ty> { 1514 def NAME : sme2_multivec_accum_add_sub_vg2<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>; 1515 1516 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", 1517 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; 1518} 1519 1520class sme2_multivec_accum_add_sub_vg4<string mnemonic, bit sz, bits<3> op, 1521 MatrixOperand matrix_ty, 1522 RegisterOperand vector_ty> 1523 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b1, op, matrix_ty, vector_ty> { 1524 bits<3> Zm; 1525 let Inst{9-7} = Zm; 1526 let Inst{6} = 0b0; 1527} 1528 1529multiclass sme2_multivec_accum_add_sub_vg4<string mnemonic, bits<4> op, 1530 MatrixOperand matrix_ty, 1531 RegisterOperand vector_ty> { 1532 def NAME : sme2_multivec_accum_add_sub_vg4<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>; 1533 1534 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", 1535 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; 1536} 1537 1538//===----------------------------------------------------------------------===// 1539// SME2 Multi-vector - Multiple and Single SVE Destructive 1540// Two and Four registers 1541 1542class sme2_sve_destructive_vector_vg2_single<bits<2> sz, bits<7> op, 1543 RegisterOperand vector_ty, 1544 ZPRRegOp zpr_ty, 1545 string mnemonic> 1546 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), 1547 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1548 "", []>, Sched<[]> { 1549 bits<4> Zm; 1550 bits<4> Zdn; 1551 let Inst{31-24} = 0b11000001; 1552 let Inst{23-22} = sz; 1553 let Inst{21-20} = 0b10; 1554 let Inst{19-16} = Zm; 1555 let Inst{15-11} = 0b10100; 1556 let Inst{10-5} = op{6-1}; 1557 let Inst{4-1} = Zdn; 1558 let Inst{0} = op{0}; 1559 1560 let Constraints = "$Zdn = $_Zdn"; 1561} 1562 1563multiclass sme2_fp_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { 1564 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; 1565 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; 1566 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; 1567} 1568 1569multiclass sme2_int_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { 1570 def _B : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_b_mul_r, ZPR4b8, mnemonic>; 1571 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; 1572 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; 1573 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; 1574} 1575 1576// SME2.1 fmax/fmin instructions. 1577multiclass sme2p1_bf_max_min_vector_vg2_single<string mnemonic, bits<7>op> { 1578 def _H : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_h_mul_r, 1579 ZPR4b16, mnemonic>; 1580} 1581 1582class sme2_sve_destructive_vector_vg4_single<bits<2> sz, bits<7> op, 1583 RegisterOperand vector_ty, 1584 ZPRRegOp zpr_ty, 1585 string mnemonic> 1586 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), 1587 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1588 "", []>, Sched<[]> { 1589 bits<4> Zm; 1590 bits<3> Zdn; 1591 let Inst{31-24} = 0b11000001; 1592 let Inst{23-22} = sz; 1593 let Inst{21-20} = 0b10; 1594 let Inst{19-16} = Zm; 1595 let Inst{15-11} = 0b10101; 1596 let Inst{10-5} = op{6-1}; 1597 let Inst{4-2} = Zdn; 1598 let Inst{1} = 0b0; 1599 let Inst{0} = op{0}; 1600 1601 let Constraints = "$Zdn = $_Zdn"; 1602} 1603 1604multiclass sme2_fp_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { 1605 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; 1606 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; 1607 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; 1608} 1609 1610multiclass sme2_int_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { 1611 def _B : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_b_mul_r, ZPR4b8, mnemonic>; 1612 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; 1613 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; 1614 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; 1615} 1616 1617// SME2.1 fmax/fmin instructions. 1618multiclass sme2p1_bf_max_min_vector_vg4_single<string mnemonic, bits<7>op> { 1619 def _H : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_h_mul_r, 1620 ZPR4b16, mnemonic>; 1621} 1622 1623class sme2_sve_destructive_vector_vg2_multi<bits<2> sz, bits<7> op, 1624 RegisterOperand vector_ty, 1625 string mnemonic> 1626 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), 1627 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1628 "", []>, Sched<[]> { 1629 bits<4> Zm; 1630 bits<4> Zdn; 1631 let Inst{31-24} = 0b11000001; 1632 let Inst{23-22} = sz; 1633 let Inst{21} = 0b1; 1634 let Inst{20-17} = Zm; 1635 let Inst{16-11} = 0b010110; 1636 let Inst{10-5} = op{6-1}; 1637 let Inst{4-1} = Zdn; 1638 let Inst{0} = op{0}; 1639 1640 let Constraints = "$Zdn = $_Zdn"; 1641} 1642 1643multiclass sme2_fp_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { 1644 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; 1645 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; 1646 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; 1647} 1648 1649multiclass sme2_int_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { 1650 def _B : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_b_mul_r, mnemonic>; 1651 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; 1652 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; 1653 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; 1654} 1655 1656// SME2.1 fmax/fmin instructions. 1657multiclass sme2p1_bf_max_min_vector_vg2_multi<string mnemonic, bits<7>op> { 1658 def _H : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_h_mul_r, 1659 mnemonic>; 1660} 1661 1662class sme2_sve_destructive_vector_vg4_multi<bits<2> sz, bits<7> op, 1663 RegisterOperand vector_ty, 1664 string mnemonic> 1665 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), 1666 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1667 "", []>, Sched<[]> { 1668 bits<3> Zm; 1669 bits<3> Zdn; 1670 let Inst{31-24} = 0b11000001; 1671 let Inst{23-22} = sz; 1672 let Inst{21} = 0b1; 1673 let Inst{20-18} = Zm; 1674 let Inst{17-11} = 0b0010111; 1675 let Inst{10-5} = op{6-1}; 1676 let Inst{4-2} = Zdn; 1677 let Inst{1} = 0b0; 1678 let Inst{0} = op{0}; 1679 1680 let Constraints = "$Zdn = $_Zdn"; 1681} 1682 1683multiclass sme2_fp_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { 1684 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; 1685 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; 1686 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; 1687} 1688 1689multiclass sme2_int_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { 1690 def _B : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_b_mul_r, mnemonic>; 1691 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; 1692 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; 1693 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; 1694} 1695 1696// SME2.1 fmax/fmin instructions. 1697multiclass sme2p1_bf_max_min_vector_vg4_multi<string mnemonic, bits<7>op> { 1698 def _H : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_h_mul_r, 1699 mnemonic>; 1700} 1701 1702//===----------------------------------------------------------------------===// 1703// SME2 Multi-vector - Index/Single/Multi Array Vectors FMA sources 1704 1705class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty, 1706 RegisterOperand multi_vector_ty, 1707 string mnemonic, string vg_acronym=""> 1708 : I<(outs MatrixOp32:$ZAda), 1709 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm, multi_vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 1710 mnemonic, "\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm$i3", 1711 "", []>, Sched<[]> { 1712 bits<4> Zm; 1713 bits<2> Rv; 1714 let Inst{31-24} = 0b11000001; 1715 let Inst{23-22} = op0; 1716 let Inst{21} = 0b0; 1717 let Inst{20} = !if(!eq(vg_acronym, ""), 0, 1); 1718 let Inst{19-16} = Zm; 1719 let Inst{14-13} = Rv; 1720 let Inst{12} = 0b1; 1721 let Inst{4-3} = op; 1722 1723 let Constraints = "$ZAda = $_ZAda"; 1724} 1725 1726multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1727 def _S : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16, 1728 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { 1729 bits<3> i3; 1730 bits<5> Zn; 1731 bits<3> imm; 1732 let Inst{15} = i3{2}; 1733 let Inst{11-10} = i3{1-0}; 1734 let Inst{9-5} = Zn; 1735 let Inst{2-0} = imm; 1736 } 1737 1738 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1739 1740 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>; 1741} 1742 1743class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op> 1744 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZ_h_mul_r, 1745 mnemonic, "vgx2"> { 1746 bits<3> i3; 1747 bits<4> Zn; 1748 bits<2> imm; 1749 let Inst{15} = 0b0; 1750 let Inst{11-10} = i3{2-1}; 1751 let Inst{9-6} = Zn; 1752 let Inst{5} = 0b0; 1753 let Inst{2} = i3{0}; 1754 let Inst{1-0} = imm; 1755} 1756 1757multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1758 def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1759 1760 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1761 1762 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; 1763 1764 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1765 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1766} 1767 1768multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1769 def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1770 1771 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1772 1773 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; 1774 1775 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1776 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1777} 1778 1779class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op> 1780 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZZZ_h_mul_r, 1781 mnemonic, "vgx4"> { 1782 bits<3> i3; 1783 bits<3> Zn; 1784 bits<2> imm; 1785 let Inst{15} = 0b1; 1786 let Inst{11-10} = i3{2-1}; 1787 let Inst{9-7} = Zn; 1788 let Inst{6-5} = 0b00; 1789 let Inst{2} = i3{0}; 1790 let Inst{1-0} = imm; 1791} 1792 1793multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1794 def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1795 1796 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1797 1798 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; 1799 1800 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1801 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1802} 1803 1804multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1805 def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1806 1807 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1808 1809 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; 1810 1811 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1812 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1813} 1814 1815class sme2_mla_long_array<bits<2>op0, bits<2> op, Operand index_ty, 1816 RegisterOperand first_vector_ty, 1817 RegisterOperand second_vector_ty, 1818 string mnemonic, string vg_acronym=""> 1819 : I<(outs MatrixOp32:$ZAda), 1820 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, 1821 index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm), 1822 mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm", 1823 "", []> , Sched<[]> { 1824 bits<2> Rv; 1825 let Inst{31-24} = 0b11000001; 1826 let Inst{23-22} = op0; 1827 let Inst{21} = 0b1; 1828 let Inst{15} = 0b0; 1829 let Inst{14-13} = Rv; 1830 let Inst{12-11} = 0b01; 1831 let Inst{10} = !if(!eq(vg_acronym, ""), 1, 0); 1832 let Inst{4-3} = op; 1833 1834 let Constraints = "$ZAda = $_ZAda"; 1835} 1836 1837multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1838 def _S : sme2_mla_long_array<op0, op, uimm3s2range, ZPR16, ZPR4b16, 1839 mnemonic> , SMEPseudo2Instr<NAME # _S, 1>{ 1840 bits<4> Zm; 1841 bits<5> Zn; 1842 bits<3> imm; 1843 let Inst{20} = 0b0; 1844 let Inst{19-16} = Zm; 1845 let Inst{9-5} = Zn; 1846 let Inst{2-0} = imm; 1847 } 1848 1849 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>; 1850 1851 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>; 1852} 1853 1854class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op, 1855 RegisterOperand first_vector_ty, 1856 string mnemonic, string vg_acronym> 1857 : sme2_mla_long_array<op0, op, uimm2s2range, first_vector_ty, ZPR4b16, 1858 mnemonic, vg_acronym> { 1859 bits<4> Zm; 1860 bits<5> Zn; 1861 bits<2> imm; 1862 let Inst{20} = vg4; 1863 let Inst{19-16} = Zm; 1864 let Inst{9-5} = Zn; 1865 let Inst{2} = 0b0; 1866 let Inst{1-0} = imm; 1867} 1868 1869multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1870 def _S : sme2_mla_long_array_vg24_single<0b00, 0b0, op, ZZ_h, mnemonic, 1871 "vgx2">, SMEPseudo2Instr<NAME # _S, 1>; 1872 1873 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; 1874 1875 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>; 1876 1877 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1878 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1879} 1880 1881multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1882 def _S : sme2_mla_long_array_vg24_single<0b01, 0b0, op, ZZ_h, mnemonic, 1883 "vgx2">, SMEPseudo2Instr<NAME # _S, 1>; 1884 1885 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; 1886 1887 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; 1888 1889 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1890 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1891} 1892 1893multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1894 def _S : sme2_mla_long_array_vg24_single<0b00, 0b1, op, ZZZZ_h, mnemonic, 1895 "vgx4">, SMEPseudo2Instr<NAME # _S, 1>; 1896 1897 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; 1898 1899 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>; 1900 1901 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1902 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1903} 1904 1905multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1906 def _S : sme2_mla_long_array_vg24_single<0b01, 0b1, op, ZZZZ_h, mnemonic, 1907 "vgx4">, SMEPseudo2Instr<NAME # _S, 1>; 1908 1909 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; 1910 1911 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; 1912 1913 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1914 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1915} 1916class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<2> op> 1917 : sme2_mla_long_array<op0, op, uimm2s2range, ZZ_h_mul_r, ZZ_h_mul_r, mnemonic, 1918 "vgx2"> { 1919 bits<4> Zm; 1920 bits<4> Zn; 1921 bits<2> imm; 1922 let Inst{20-17} = Zm; 1923 let Inst{16} = 0b0; 1924 let Inst{9-6} = Zn; 1925 let Inst{5} = 0b0; 1926 let Inst{2} = 0b0; 1927 let Inst{1-0} = imm; 1928} 1929 1930multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1931 def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1932 1933 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; 1934 1935 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; 1936 1937 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1938 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; 1939} 1940 1941multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1942 def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1943 1944 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; 1945 1946 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; 1947 1948 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", 1949 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; 1950} 1951 1952class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<2> op> 1953 : sme2_mla_long_array<op0, op, uimm2s2range, ZZZZ_h_mul_r, ZZZZ_h_mul_r, mnemonic, 1954 "vgx4"> { 1955 bits<3> Zm; 1956 bits<3> Zn; 1957 bits<2> imm; 1958 let Inst{20-18} = Zm; 1959 let Inst{17} = 0b0; 1960 let Inst{16} = 0b1; 1961 let Inst{9-7} = Zn; 1962 let Inst{6-5} = 0b00; 1963 let Inst{2} = 0b0; 1964 let Inst{1-0} = imm; 1965} 1966 1967multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1968 def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1969 1970 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; 1971 1972 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; 1973 1974 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1975 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; 1976} 1977 1978multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1979 def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1980 1981 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; 1982 1983 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; 1984 1985 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", 1986 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; 1987} 1988 1989//===----------------------------------------------------------------------===// 1990class sme2_frint_cvt_vg2_multi<bits<2>sz, bits<5>op, RegisterOperand first_ty, 1991 RegisterOperand second_ty, string mnemonic> 1992 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 1993 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 1994 bits<4> Zn; 1995 bits<4> Zd; 1996 let Inst{31-24} = 0b11000001; 1997 let Inst{23-22} = sz; 1998 let Inst{21-20} = 0b10; 1999 let Inst{19-16} = op{4-1}; 2000 let Inst{15-10} = 0b111000; 2001 let Inst{9-6} = Zn; 2002 let Inst{5} = op{0}; 2003 let Inst{4-1} = Zd; 2004 let Inst{0} = 0b0; 2005} 2006 2007// SME2 multi-vec FP to int convert two registers 2008// SME2 multi-vec int to FP two registers 2009multiclass sme2_fp_cvt_vg2_multi<string mnemonic, bits<5> op> { 2010 def NAME : sme2_frint_cvt_vg2_multi<0b00, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; 2011} 2012 2013// SME2 multi-vec FRINT two registers 2014multiclass sme2_frint_vector_vg2_multi<string mnemonic, bits<5> op> { 2015 def _S : sme2_frint_cvt_vg2_multi<0b10, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; 2016} 2017 2018class sme2_frint_zip_cvt_vg4_multi<bits<2>sz, bits<7>op, RegisterOperand first_ty, 2019 RegisterOperand second_ty, string mnemonic> 2020 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2021 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2022 bits<3> Zn; 2023 bits<3> Zd; 2024 let Inst{31-24} = 0b11000001; 2025 let Inst{23-22} = sz; 2026 let Inst{21-20} = 0b11; 2027 let Inst{19-16} = op{6-3}; 2028 let Inst{15-10} = 0b111000; 2029 let Inst{9-7} = Zn; 2030 let Inst{6-5} = op{2-1}; 2031 let Inst{4-2} = Zd; 2032 let Inst{1} = op{0}; 2033 let Inst{0} = 0b0; 2034} 2035 2036// SME2 multi-vec FP to int convert four registers 2037// SME2 multi-vec int to FP four registers 2038multiclass sme2_fp_cvt_vg4_multi<string mnemonic, bits<7> op> { 2039 def NAME : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, mnemonic>; 2040} 2041 2042// SME2 multi-vec quadwords ZIP four registers 2043multiclass sme2_zip_vector_vg4<string mnemonic, bits<7> op> { 2044 def _B : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_b_mul_r, ZZZZ_b_mul_r, 2045 mnemonic>; 2046 def _H : sme2_frint_zip_cvt_vg4_multi<0b01, op, ZZZZ_h_mul_r, ZZZZ_h_mul_r, 2047 mnemonic>; 2048 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, 2049 mnemonic>; 2050 def _D : sme2_frint_zip_cvt_vg4_multi<0b11, op, ZZZZ_d_mul_r, ZZZZ_d_mul_r, 2051 mnemonic>; 2052} 2053 2054// SME2 multi-vec quadwords ZIP four registers 2055multiclass sme2_zip_vector_vg4_Q<string mnemonic, bits<7> op> { 2056 def NAME: sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_q_mul_r, ZZZZ_q_mul_r, 2057 mnemonic>; 2058} 2059 2060// SME2 multi-vec FRINT four registers 2061multiclass sme2_frint_vector_vg4_multi<string mnemonic, bits<7> op> { 2062 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, 2063 mnemonic>; 2064} 2065 2066class sme2_cvt_vg2_single<string mnemonic, bits<4> op> 2067 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn), 2068 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2069 bits<4> Zn; 2070 bits<5> Zd; 2071 let Inst{31-23} = 0b110000010; 2072 let Inst{22} = op{3}; 2073 let Inst{21-18} = 0b1000; 2074 let Inst{17-16} = op{2-1}; 2075 let Inst{15-10} = 0b111000; 2076 let Inst{9-6} = Zn; 2077 let Inst{5} = op{0}; 2078 let Inst{4-0} = Zd; 2079} 2080 2081// SME2 multi-vec FP down convert two registers 2082// SME2 multi-vec int down convert two registers 2083multiclass sme2_cvt_vg2_single<string mnemonic, bits<4> op, ValueType out_vt, 2084 ValueType in_vt, SDPatternOperator intrinsic> { 2085 def NAME : sme2_cvt_vg2_single<mnemonic, op>; 2086 def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, out_vt, in_vt>; 2087} 2088 2089class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty, 2090 RegisterOperand second_ty, string mnemonic> 2091 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2092 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2093 bits<5> Zn; 2094 bits<4> Zd; 2095 let Inst{31-24} = 0b11000001; 2096 let Inst{23-22} = sz; 2097 let Inst{21-19} = 0b100; 2098 let Inst{18-16} = op; 2099 let Inst{15-10} = 0b111000; 2100 let Inst{9-5} = Zn; 2101 let Inst{4-1} = Zd; 2102 let Inst{0} = u; 2103} 2104 2105// SME2 multi-vec unpack two registers 2106multiclass sme2_unpk_vector_vg2<string mnemonic, bit u> { 2107 def _H : sme2_cvt_unpk_vector_vg2<0b01, 0b101, u, ZZ_h_mul_r, ZPR8, mnemonic>; 2108 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b101, u, ZZ_s_mul_r, ZPR16, mnemonic>; 2109 def _D : sme2_cvt_unpk_vector_vg2<0b11, 0b101, u, ZZ_d_mul_r, ZPR32, mnemonic>; 2110} 2111 2112// SME2.1 multi-vec convert two registers 2113multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> { 2114 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b000, l, ZZ_s_mul_r, ZPR16, mnemonic>; 2115} 2116 2117class sme2_cvt_vg4_single<bit sz, bits<3> op, RegisterOperand first_ty, 2118 RegisterOperand second_ty, string mnemonic> 2119 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2120 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2121 bits<3> Zn; 2122 bits<5> Zd; 2123 let Inst{31-24} = 0b11000001; 2124 let Inst{23} = sz; 2125 let Inst{22} = op{2}; 2126 let Inst{21-10} = 0b110011111000; 2127 let Inst{9-7} = Zn; 2128 let Inst{6-5} = op{1-0}; 2129 let Inst{4-0} = Zd; 2130} 2131 2132// SME2 multi-vec int down convert four registers 2133multiclass sme2_int_cvt_vg4_single<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2134 def _StoB : sme2_cvt_vg4_single<0, op, ZPR8, ZZZZ_s_mul_r, mnemonic>; 2135 def _DtoH : sme2_cvt_vg4_single<1, op, ZPR16, ZZZZ_d_mul_r, mnemonic>; 2136 2137 def : SME2_Cvt_VG4_Pat<NAME # _StoB, intrinsic, nxv16i8, nxv4i32>; 2138 def : SME2_Cvt_VG4_Pat<NAME # _DtoH, intrinsic, nxv8i16, nxv2i64>; 2139} 2140 2141class sme2_unpk_vector_vg4<bits<2>sz, bit u, RegisterOperand first_ty, 2142 RegisterOperand second_ty, string mnemonic> 2143 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2144 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2145 bits<4> Zn; 2146 bits<3> Zd; 2147 let Inst{31-24} = 0b11000001; 2148 let Inst{23-22} = sz; 2149 let Inst{21-10} = 0b110101111000; 2150 let Inst{9-6} = Zn; 2151 let Inst{5} = 0b0; 2152 let Inst{4-2} = Zd; 2153 let Inst{1} = 0b0; 2154 let Inst{0} = u; 2155} 2156 2157// SME2 multi-vec UNPK four registers 2158multiclass sme2_unpk_vector_vg4<string mnemonic, bit u> { 2159 def _H : sme2_unpk_vector_vg4<0b01, u, ZZZZ_h_mul_r, ZZ_b_mul_r, mnemonic>; 2160 def _S : sme2_unpk_vector_vg4<0b10, u, ZZZZ_s_mul_r, ZZ_h_mul_r, mnemonic>; 2161 def _D : sme2_unpk_vector_vg4<0b11, u, ZZZZ_d_mul_r, ZZ_s_mul_r, mnemonic>; 2162} 2163 2164//===----------------------------------------------------------------------===// 2165// SME2 multi-vec CLAMP registers 2166 2167class sme2_clamp_vector_vg24_multi<bits<2> sz, bits<3> op1, bit u, 2168 RegisterOperand multi_vector_ty, 2169 ZPRRegOp vector_ty, string mnemonic> 2170 : I<(outs multi_vector_ty:$Zd), 2171 (ins multi_vector_ty:$_Zd, vector_ty:$Zn, vector_ty:$Zm), 2172 mnemonic, "\t$Zd, $Zn, $Zm", 2173 "", []>, Sched<[]>{ 2174 bits<5> Zm; 2175 bits<5> Zn; 2176 let Inst{31-24} = 0b11000001; 2177 let Inst{23-22} = sz; 2178 let Inst{21} = 0b1; 2179 let Inst{20-16} = Zm; 2180 let Inst{15-13} = 0b110; 2181 let Inst{12-10} = op1; 2182 let Inst{9-5} = Zn; 2183 let Inst{0} = u; 2184 2185 let Constraints = "$Zd = $_Zd"; 2186} 2187 2188class sme2_clamp_vector_vg2_multi<bits<2> sz, bits<3> op1, bit u, 2189 RegisterOperand multi_vector_ty, 2190 ZPRRegOp vector_ty, string mnemonic> 2191 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, 2192 mnemonic>{ 2193 bits<4> Zd; 2194 let Inst{4-1} = Zd; 2195} 2196 2197multiclass sme2_fp_clamp_vector_vg2_multi<string mnemonic>{ 2198 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b000, 0b0, ZZ_h_mul_r, ZPR16, mnemonic>; 2199 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b000, 0b0, ZZ_s_mul_r, ZPR32, mnemonic>; 2200 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b000, 0b0, ZZ_d_mul_r, ZPR64, mnemonic>; 2201} 2202 2203multiclass sme2_int_clamp_vector_vg2_multi<string mnemonic, bit u>{ 2204 def _B : sme2_clamp_vector_vg2_multi<0b00, 0b001, u, ZZ_b_mul_r, ZPR8, mnemonic>; 2205 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b001, u, ZZ_h_mul_r, ZPR16, mnemonic>; 2206 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b001, u, ZZ_s_mul_r, ZPR32, mnemonic>; 2207 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b001, u, ZZ_d_mul_r, ZPR64, mnemonic>; 2208} 2209 2210// SME2.1 multi-vec FCLAMP two registers 2211multiclass sme2p1_bfclamp_vector_vg2_multi<string mnemonic> { 2212 def _H : sme2_clamp_vector_vg2_multi<0b00, 0b000, 0b0, ZZ_h_mul_r, ZPR16, 2213 mnemonic>; 2214} 2215 2216class sme2_clamp_vector_vg4_multi<bits<2> sz, bits<3> op1, bit u, 2217 RegisterOperand multi_vector_ty, 2218 ZPRRegOp vector_ty, string mnemonic> 2219 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, 2220 mnemonic>{ 2221 bits<3> Zd; 2222 let Inst{4-2} = Zd; 2223 let Inst{1} = 0b0; 2224} 2225 2226multiclass sme2_fp_clamp_vector_vg4_multi<string mnemonic>{ 2227 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, mnemonic>; 2228 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b010, 0b0, ZZZZ_s_mul_r, ZPR32, mnemonic>; 2229 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b010, 0b0, ZZZZ_d_mul_r, ZPR64, mnemonic>; 2230} 2231 2232multiclass sme2_int_clamp_vector_vg4_multi<string mnemonic, bit u>{ 2233 def _B : sme2_clamp_vector_vg4_multi<0b00, 0b011, u, ZZZZ_b_mul_r, ZPR8, mnemonic>; 2234 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b011, u, ZZZZ_h_mul_r, ZPR16, mnemonic>; 2235 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b011, u, ZZZZ_s_mul_r, ZPR32, mnemonic>; 2236 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b011, u, ZZZZ_d_mul_r, ZPR64, mnemonic>; 2237} 2238 2239// SME2.1 multi-vec FCLAMP four registers 2240multiclass sme2p1_bfclamp_vector_vg4_multi<string mnemonic> { 2241 def _H : sme2_clamp_vector_vg4_multi<0b00, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, 2242 mnemonic>; 2243} 2244 2245// SME2 multi-vec ZIP two registers 2246class sme2_zip_vector_vg2<bits<2> sz, bit q, bit u, 2247 RegisterOperand multi_vector_ty, 2248 ZPRRegOp vector_ty, string mnemonic> 2249 : I<(outs multi_vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm), 2250 mnemonic, "\t$Zd, $Zn, $Zm", 2251 "", []>, Sched<[]>{ 2252 bits<4> Zd; 2253 bits<5> Zm; 2254 bits<5> Zn; 2255 let Inst{31-24} = 0b11000001; 2256 let Inst{23-22} = sz; 2257 let Inst{21} = 0b1; 2258 let Inst{20-16} = Zm; 2259 let Inst{15-11} = 0b11010; 2260 let Inst{10} = q; 2261 let Inst{9-5} = Zn; 2262 let Inst{4-1} = Zd; 2263 let Inst{0} = u; 2264} 2265 2266multiclass sme2_zip_vector_vg2<string mnemonic, bit op> { 2267 def _B : sme2_zip_vector_vg2<0b00, 0b0, op, ZZ_b_mul_r, ZPR8, mnemonic>; 2268 def _H : sme2_zip_vector_vg2<0b01, 0b0, op, ZZ_h_mul_r, ZPR16, mnemonic>; 2269 def _S : sme2_zip_vector_vg2<0b10, 0b0, op, ZZ_s_mul_r, ZPR32, mnemonic>; 2270 def _D : sme2_zip_vector_vg2<0b11, 0b0, op, ZZ_d_mul_r, ZPR64, mnemonic>; 2271 def _Q : sme2_zip_vector_vg2<0b00, 0b1, op, ZZ_q_mul_r, ZPR128, mnemonic>; 2272} 2273 2274//===----------------------------------------------------------------------===// 2275// SME2 Dot Products and MLA 2276 2277class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty, 2278 RegisterOperand multi_vector_ty, 2279 ZPRRegOp vector_ty, Operand index_ty, 2280 string mnemonic> 2281 : I<(outs matrix_ty:$ZAda), 2282 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2283 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), 2284 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i", 2285 "", []>, Sched<[]> { 2286 bits<4> Zm; 2287 bits<2> Rv; 2288 bits<4> Zn; 2289 bits<3> imm3; 2290 let Inst{31-23} = 0b110000010; 2291 let Inst{22} = sz; 2292 let Inst{21-20} = 0b01; 2293 let Inst{19-16} = Zm; 2294 let Inst{15} = 0b0; 2295 let Inst{14-13} = Rv; 2296 let Inst{12-10} = op{5-3}; 2297 let Inst{9-6} = Zn; 2298 let Inst{5-3} = op{2-0}; 2299 let Inst{2-0} = imm3; 2300 2301 let Constraints = "$ZAda = $_ZAda"; 2302} 2303 2304// SME2 multi-vec ternary indexed two registers 32-bit 2305multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<4> op, 2306 RegisterOperand multi_vector_ty, 2307 ZPRRegOp vector_ty, ValueType vt, 2308 SDPatternOperator intrinsic> { 2309 def NAME : sme2_multi_vec_array_vg2_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty, 2310 VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2311 bits<2> i; 2312 let Inst{11-10} = i; 2313 } 2314 2315 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; 2316 2317 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; 2318 2319 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2320 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2321 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; 2322} 2323 2324// SME2.1 multi-vec ternary indexed two registers 16-bit 2325multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> op> { 2326 def NAME : sme2_multi_vec_array_vg2_index<0b0, {0b1,?,?,op,?}, MatrixOp16, 2327 ZZ_h_mul_r, ZPR4b16, 2328 VectorIndexH, mnemonic> { 2329 bits<3> i; 2330 let Inst{11-10} = i{2-1}; 2331 let Inst{3} = i{0}; 2332 } 2333 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2334 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2335 ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; 2336} 2337 2338// SME2 multi-vec ternary indexed two registers 64-bit 2339 2340class sme2_multi_vec_array_vg2_index_64b<bits<2> op, 2341 RegisterOperand multi_vector_ty, 2342 ZPRRegOp vector_ty, 2343 string mnemonic> 2344 : I<(outs MatrixOp64:$ZAda), 2345 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2346 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 2347 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i1", 2348 "", []>, Sched<[]> { 2349 bits<4> Zm; 2350 bits<2> Rv; 2351 bits<1> i1; 2352 bits<4> Zn; 2353 bits<3> imm3; 2354 let Inst{31-20} = 0b110000011101; 2355 let Inst{19-16} = Zm; 2356 let Inst{15} = 0b0; 2357 let Inst{14-13} = Rv; 2358 let Inst{12-11} = 0b00; 2359 let Inst{10} = i1; 2360 let Inst{9-6} = Zn; 2361 let Inst{5} = 0b0; 2362 let Inst{4-3} = op; 2363 let Inst{2-0} = imm3; 2364 2365 let Constraints = "$ZAda = $_ZAda"; 2366} 2367 2368multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op, 2369 RegisterOperand multi_vector_ty, 2370 ZPRRegOp vector_ty, ValueType vt, 2371 SDPatternOperator intrinsic> { 2372 def NAME : sme2_multi_vec_array_vg2_index_64b<op, multi_vector_ty, vector_ty, 2373 mnemonic>, SMEPseudo2Instr<NAME, 1>; 2374 2375 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; 2376 2377 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexD32b_timm, tileslice16>; 2378 2379 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", 2380 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2381 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; 2382} 2383 2384class sme2_multi_vec_array_vg4_index<bit sz, bits<6> op, MatrixOperand matrix_ty, 2385 RegisterOperand multi_vector_ty, 2386 ZPRRegOp vector_ty, Operand index_ty, 2387 string mnemonic> 2388 : I<(outs matrix_ty:$ZAda), 2389 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2390 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), 2391 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i", 2392 "", []>, Sched<[]> { 2393 bits<4> Zm; 2394 bits<2> Rv; 2395 bits<3> Zn; 2396 bits<3> imm3; 2397 let Inst{31-23} = 0b110000010; 2398 let Inst{22} = sz; 2399 let Inst{21-20} = 0b01; 2400 let Inst{19-16} = Zm; 2401 let Inst{15} = 0b1; 2402 let Inst{14-13} = Rv; 2403 let Inst{12-10} = op{5-3}; 2404 let Inst{9-7} = Zn; 2405 let Inst{6} = 0b0; 2406 let Inst{5-3} = op{2-0}; 2407 let Inst{2-0} = imm3; 2408 2409 let Constraints = "$ZAda = $_ZAda"; 2410} 2411 2412// SME2 multi-vec ternary indexed four registers 32-bit 2413multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op, 2414 RegisterOperand multi_vector_ty, 2415 ZPRRegOp vector_ty, ValueType vt, 2416 SDPatternOperator intrinsic> { 2417 def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, 2418 vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2419 bits<2> i; 2420 let Inst{11-10} = i; 2421 } 2422 2423 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; 2424 2425 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; 2426 2427 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2428 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2429 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; 2430} 2431 2432// SME2.1 multi-vec ternary indexed four registers 16-bit 2433multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<2> op> { 2434 def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16, 2435 ZZZZ_h_mul_r, ZPR4b16, 2436 VectorIndexH, mnemonic>{ 2437 bits<3> i; 2438 let Inst{11-10} = i{2-1}; 2439 let Inst{3} = i{0}; 2440 } 2441 2442 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2443 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2444 sme_elm_idx0_7:$imm3, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; 2445} 2446 2447// SME2 multi-vec ternary indexed four registers 64-bit 2448class sme2_multi_vec_array_vg4_index_64b<bits<3> op, 2449 RegisterOperand multi_vector_ty, 2450 ZPRRegOp vector_ty, 2451 string mnemonic> 2452 : I<(outs MatrixOp64:$ZAda), 2453 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2454 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 2455 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i1", 2456 "", []>, Sched<[]> { 2457 bits<4> Zm; 2458 bits<2> Rv; 2459 bits<1> i1; 2460 bits<3> Zn; 2461 bits<3> imm3; 2462 let Inst{31-20} = 0b110000011101; 2463 let Inst{19-16} = Zm; 2464 let Inst{15} = 0b1; 2465 let Inst{14-13} = Rv; 2466 let Inst{12} = 0b0; 2467 let Inst{11} = op{2}; 2468 let Inst{10} = i1; 2469 let Inst{9-7} = Zn; 2470 let Inst{6-5} = 0b00; 2471 let Inst{4-3} = op{1-0}; 2472 let Inst{2-0} = imm3; 2473 2474 let Constraints = "$ZAda = $_ZAda"; 2475} 2476 2477multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op, 2478 RegisterOperand multi_vector_ty, 2479 ZPRRegOp vector_ty, ValueType vty, 2480 SDPatternOperator intrinsic> { 2481 def NAME : sme2_multi_vec_array_vg4_index_64b<op, multi_vector_ty, vector_ty, 2482 mnemonic>, SMEPseudo2Instr<NAME, 1>; 2483 2484 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; 2485 2486 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vty, VectorIndexD32b_timm, tileslice16>; 2487 2488 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", 2489 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2490 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; 2491} 2492//===----------------------------------------------------------------------===// 2493// SME2 multi-vec indexed long long MLA one source 32-bit 2494class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op> 2495 : I<(outs MatrixOp32:$ZAda), 2496 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 2497 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2498 "", []>, Sched<[]> { 2499 bits<4> Zm; 2500 bits<2> Rv; 2501 bits<4> i; 2502 bits<5> Zn; 2503 bits<2> imm2; 2504 let Inst{31-20} = 0b110000010000; 2505 let Inst{19-16} = Zm; 2506 let Inst{15} = i{3}; 2507 let Inst{14-13} = Rv; 2508 let Inst{12-10} = i{2-0}; 2509 let Inst{9-5} = Zn; 2510 let Inst{4-2} = op; 2511 let Inst{1-0} = imm2; 2512 2513 let Constraints = "$ZAda = $_ZAda"; 2514} 2515 2516// SME2 multi-vec indexed long long MLA one source 64-bit 2517 2518class sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op> 2519 : I<(outs MatrixOp64:$ZAda), 2520 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR16:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 2521 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2522 "", []>, Sched<[]> { 2523 bits<4> Zm; 2524 bits<2> Rv; 2525 bits<3> i; 2526 bits<5> Zn; 2527 bits<2> imm2; 2528 let Inst{31-20} = 0b110000011000; 2529 let Inst{19-16} = Zm; 2530 let Inst{15} = i{2}; 2531 let Inst{14-13} = Rv; 2532 let Inst{12} = 0b0; 2533 let Inst{11-10} = i{1-0}; 2534 let Inst{9-5} = Zn; 2535 let Inst{4-3} = op; 2536 let Inst{2} = 0b0; 2537 let Inst{1-0} = imm2; 2538 2539 let Constraints = "$ZAda = $_ZAda"; 2540} 2541 2542class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op, 2543 RegisterOperand vector_ty, 2544 string mnemonic> 2545 : I<(outs MatrixOp32:$ZAda), 2546 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2547 vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 2548 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2549 "", []>, Sched<[]> { 2550 bits<4> Zm; 2551 bits<2> Rv; 2552 bits<4> i; 2553 bit imm; 2554 let Inst{31-20} = 0b110000010001; 2555 let Inst{19-16} = Zm; 2556 let Inst{15} = vg4; 2557 let Inst{14-13} = Rv; 2558 let Inst{12} = 0b0; 2559 let Inst{11-10} = i{3-2}; 2560 let Inst{5-3} = op; 2561 let Inst{2-1} = i{1-0}; 2562 let Inst{0} = imm; 2563 2564 let Constraints = "$ZAda = $_ZAda"; 2565} 2566 2567//SME2 multi-vec indexed long long MLA two sources 32-bit 2568 2569multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<3> op> { 2570 def NAME: sme2_mla_ll_array_vg24_index_32b<0b0, op, ZZ_b_mul_r, mnemonic> { 2571 bits<4> Zn; 2572 let Inst{9-6} = Zn; 2573 } 2574 2575 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2576 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>; 2577} 2578 2579// SME2 multi-vec indexed long long MLA four sources 32-bit 2580 2581multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<3> op> { 2582 def NAME: sme2_mla_ll_array_vg24_index_32b<0b1, op, ZZZZ_b_mul_r, mnemonic> { 2583 bits<3> Zn; 2584 let Inst{9-7} = Zn; 2585 let Inst{6} = 0b0; 2586 } 2587 2588 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2589 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>; 2590} 2591class sme2_mla_ll_array_vg24_index_64b<bit vg4, bits<2> op, 2592 RegisterOperand vector_ty, 2593 string mnemonic> 2594 : I<(outs MatrixOp64:$ZAda), 2595 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2596 vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 2597 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2598 "", []>, Sched<[]> { 2599 bits<4> Zm; 2600 bits<2> Rv; 2601 bits<3> i; 2602 bit imm; 2603 let Inst{31-20} = 0b110000011001; 2604 let Inst{19-16} = Zm; 2605 let Inst{15} = vg4; 2606 let Inst{14-13} = Rv; 2607 let Inst{12-11} = 0b00; 2608 let Inst{10} = i{2}; 2609 let Inst{5} = 0b0; 2610 let Inst{4-3} = op; 2611 let Inst{2-1} = i{1-0}; 2612 let Inst{0} = imm; 2613 2614 let Constraints = "$ZAda = $_ZAda"; 2615} 2616 2617// SME2 multi-vec indexed long long MLA two sources 64-bit 2618 2619multiclass sme2_mla_ll_array_vg2_index_64b<string mnemonic, bits<2> op> { 2620 def NAME: sme2_mla_ll_array_vg24_index_64b<0b0, op, ZZ_h_mul_r, mnemonic>{ 2621 bits<4> Zn; 2622 let Inst{9-6} = Zn; 2623 } 2624 2625 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2626 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; 2627} 2628 2629// SME2 multi-vec indexed long long MLA four sources 64-bit 2630 2631multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op> { 2632 def NAME: sme2_mla_ll_array_vg24_index_64b<0b1, op, ZZZZ_h_mul_r, mnemonic>{ 2633 bits<3> Zn; 2634 let Inst{9-7} = Zn; 2635 let Inst{6} = 0b0; 2636 } 2637 2638 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2639 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; 2640} 2641 2642 2643//SME2 multiple and single vector long long FMA one source 2644 2645class sme2_mla_ll_array_single<string mnemonic, bits<4> op, 2646 MatrixOperand matrix_ty, ZPRRegOp vector_ty, 2647 ZPRRegOp zpr_ty> 2648 : I<(outs matrix_ty:$ZAda), 2649 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm, 2650 vector_ty:$Zn, zpr_ty:$Zm), 2651 mnemonic, "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2652 "", []>, Sched<[]> { 2653 bits<4> Zm; 2654 bits<2> Rv; 2655 bits<5> Zn; 2656 bits<2> imm; 2657 let Inst{31-23} = 0b110000010; 2658 let Inst{22} = op{3}; //sz 2659 let Inst{21-20} = 0b10; 2660 let Inst{19-16} = Zm; 2661 let Inst{15} = 0b0; 2662 let Inst{14-13} = Rv; 2663 let Inst{12-10} = 0b001; 2664 let Inst{9-5} = Zn; 2665 let Inst{4-2} = op{2-0}; 2666 let Inst{1-0} = imm; 2667 2668 let Constraints = "$ZAda = $_ZAda"; 2669} 2670 2671class sme2_mla_ll_array_vg24_single<bits<5> op, MatrixOperand matrix_ty, 2672 RegisterOperand vector_ty, ZPRRegOp zpr_ty, 2673 string mnemonic> 2674 : I<(outs matrix_ty:$ZAda), 2675 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2676 vector_ty:$Zn, zpr_ty:$Zm), 2677 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(op{3}, "vgx4", "vgx2") # "], $Zn, $Zm", 2678 "", []>, Sched<[]> { 2679 bits<4> Zm; 2680 bits<2> Rv; 2681 bits<5> Zn; 2682 bit imm; 2683 let Inst{31-23} = 0b110000010; 2684 let Inst{22} = op{4}; //sz 2685 let Inst{21} = 0b1; 2686 let Inst{20} = op{3}; //vg4 2687 let Inst{19-16} = Zm; 2688 let Inst{15} = 0b0; 2689 let Inst{14-13} = Rv; 2690 let Inst{12-10} = 0b000; 2691 let Inst{9-5} = Zn; 2692 let Inst{4-2} = op{2-0}; 2693 let Inst{1} = 0b0; 2694 let Inst{0} = imm; 2695 2696 let Constraints = "$ZAda = $_ZAda"; 2697} 2698 2699//SME2 single-multi long long MLA two and four sources 2700 2701multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<5> op, 2702 MatrixOperand matrix_ty, 2703 RegisterOperand multi_vector_ty, 2704 ZPRRegOp zpr_ty> { 2705 def NAME: sme2_mla_ll_array_vg24_single<op, matrix_ty, multi_vector_ty, 2706 zpr_ty, mnemonic>; 2707 2708 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm], $Zn, $Zm", 2709 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 2710} 2711 2712// SME2 multiple vectors long long MLA two sources 2713 2714class sme2_mla_ll_array_vg2_multi<bits<4> op, MatrixOperand matrix_ty, 2715 RegisterOperand vector_ty,string mnemonic> 2716 : I<(outs matrix_ty:$ZAda), 2717 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2718 vector_ty:$Zn, vector_ty:$Zm), 2719 mnemonic, "\t$ZAda[$Rv, $imm, vgx2], $Zn, $Zm", 2720 "", []>, Sched<[]> { 2721 bits<4> Zm; 2722 bits<2> Rv; 2723 bits<4> Zn; 2724 bit imm; 2725 let Inst{31-23} = 0b110000011; 2726 let Inst{22} = op{3}; // sz 2727 let Inst{21} = 0b1; 2728 let Inst{20-17} = Zm; 2729 let Inst{16-15} = 0b00; 2730 let Inst{14-13} = Rv; 2731 let Inst{12-10} = 0b000; 2732 let Inst{9-6} = Zn; 2733 let Inst{5} = 0b0; 2734 let Inst{4-2} = op{2-0}; 2735 let Inst{1} = 0b0; 2736 let Inst{0} = imm; 2737 2738 let Constraints = "$ZAda = $_ZAda"; 2739} 2740 2741multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<4> op, 2742 MatrixOperand matrix_ty, 2743 RegisterOperand vector_ty> { 2744 def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>; 2745 2746 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2747 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; 2748} 2749 2750// SME2 multiple vectors long long MLA four sources 2751 2752class sme2_mla_ll_array_vg4_multi<bits<4> op,MatrixOperand matrix_ty, 2753 RegisterOperand vector_ty, 2754 string mnemonic> 2755 : I<(outs matrix_ty:$ZAda), 2756 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2757 vector_ty:$Zn, vector_ty:$Zm), 2758 mnemonic, "\t$ZAda[$Rv, $imm, vgx4], $Zn, $Zm", 2759 "", []>, Sched<[]> { 2760 bits<3> Zm; 2761 bits<2> Rv; 2762 bits<3> Zn; 2763 bit imm; 2764 let Inst{31-23} = 0b110000011; 2765 let Inst{22} = op{3}; // sz 2766 let Inst{21} = 0b1; 2767 let Inst{20-18} = Zm; 2768 let Inst{17-15} = 0b010; 2769 let Inst{14-13} = Rv; 2770 let Inst{12-10} = 0b000; 2771 let Inst{9-7} = Zn; 2772 let Inst{6-5} = 0b00; 2773 let Inst{4-2} = op{2-0}; 2774 let Inst{1} = 0b0; 2775 let Inst{0} = imm; 2776 2777 let Constraints = "$ZAda = $_ZAda"; 2778} 2779 2780multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<4> op, 2781 MatrixOperand matrix_ty, 2782 RegisterOperand vector_ty> { 2783 def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>; 2784 2785 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2786 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; 2787} 2788 2789//===----------------------------------------------------------------------===// 2790// SME2 Outer Product and Accumulate 2791 2792multiclass sme2_int_mopx_tile<string mnemonic, bits<3> op> { 2793 def NAME : sme_int_outer_product_inst<op, 0b0, 0b1, TileOp32, ZPR16, mnemonic> { 2794 bits<2> ZAda; 2795 let Inst{1-0} = ZAda; 2796 let Inst{2} = 0b0; 2797 } 2798} 2799 2800multiclass sme2_bfp_mopx_tile<string mnemonic, bits<3> op> { 2801 def NAME : sme_outer_product_widening_inst<op, ZPR32, mnemonic>; 2802} 2803 2804//===----------------------------------------------------------------------===/// 2805// SME2 Zero Lookup Table. 2806class sme2_zero_zt<string mnemonic, bits<4> opc> 2807 : I<(outs ZTR:$ZT), (ins ), 2808 mnemonic, "\t\\{ $ZT \\}", 2809 "", []>, Sched<[]> { 2810 let Inst{31-4} = 0b1100000001001000000000000000; 2811 let Inst{3-0} = opc; 2812} 2813 2814//===----------------------------------------------------------------------===// 2815// SME2 lookup table load/store 2816class sme2_spill_fill_vector<string mnemonic, bits<8> opc> 2817 : I<!if(opc{7}, (outs ), (outs ZTR:$ZTt)), 2818 !if(opc{7}, (ins ZTR:$ZTt, GPR64sp:$Rn), (ins GPR64sp:$Rn)), 2819 mnemonic, "\t$ZTt, [$Rn]", 2820 "", []>, Sched<[]> { 2821 bits<5> Rn; 2822 let Inst{31-22} = 0b1110000100; 2823 let Inst{21-16} = opc{7-2}; 2824 let Inst{15-10} = 0b100000; 2825 let Inst{9-5} = Rn; 2826 let Inst{4-2} = 0b000; 2827 let Inst{1-0} = opc{1-0}; 2828 2829 let mayLoad = !not(opc{7}); 2830 let mayStore = opc{7}; 2831} 2832 2833//===----------------------------------------------------------------------===/// 2834// SME2 move to/from lookup table 2835class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc> 2836 : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3), 2837 mnemonic, "\t$Rt, $ZTt$imm3", 2838 "", []>, Sched<[]> { 2839 bits<3> imm3; 2840 bits<5> Rt; 2841 let Inst{31-15} = 0b11000000010011000; 2842 let Inst{14-12} = imm3; 2843 let Inst{11-5} = opc; 2844 let Inst{4-0} = Rt; 2845} 2846 2847class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc> 2848 : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt), 2849 mnemonic, "\t$ZTt$imm3, $Rt", 2850 "", []>, Sched<[]> { 2851 bits<3> imm3; 2852 bits<5> Rt; 2853 let Inst{31-15} = 0b11000000010011100; 2854 let Inst{14-12} = imm3; 2855 let Inst{11-5} = opc; 2856 let Inst{4-0} = Rt; 2857} 2858 2859//===----------------------------------------------------------------------===// 2860// SME2 lookup table expand one register 2861class sme2_luti_vector_index<bits<2> sz, bits<7> opc, RegisterOperand vector_ty, 2862 AsmVectorIndexOpnd index_ty, string mnemonic> 2863 : I<(outs vector_ty:$Zd), 2864 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 2865 mnemonic, "\t$Zd, $ZTt, $Zn$i", 2866 "", []>, Sched<[]> { 2867 bits<5> Zn; 2868 bits<5> Zd; 2869 let Inst{31-19} = 0b1100000011001; 2870 let Inst{18-14} = opc{6-2}; 2871 let Inst{13-12} = sz; 2872 let Inst{11-10} = opc{1-0}; 2873 let Inst{9-5} = Zn; 2874 let Inst{4-0} = Zd; 2875} 2876 2877class sme2_luti2_vector_index<bits<2> sz, RegisterOperand vector_ty, 2878 string mnemonic> 2879 : sme2_luti_vector_index<sz, {1,?,?,?,?,0,0}, vector_ty, VectorIndexB, mnemonic> { 2880 bits<4> i; 2881 let Inst{17-14} = i; 2882} 2883 2884multiclass sme2_luti2_vector_index<string mnemonic> { 2885 def _B : sme2_luti2_vector_index<0b00, ZPR8, mnemonic>; 2886 def _H : sme2_luti2_vector_index<0b01, ZPR16, mnemonic>; 2887 def _S : sme2_luti2_vector_index<0b10, ZPR32, mnemonic>; 2888} 2889 2890class sme2_luti4_vector_index<bits<2> sz, RegisterOperand vector_ty, 2891 string mnemonic> 2892 : sme2_luti_vector_index<sz, {0,1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { 2893 bits<3> i; 2894 let Inst{16-14} = i; 2895} 2896 2897multiclass sme2_luti4_vector_index<string mnemonic> { 2898 def _B : sme2_luti4_vector_index<0b00, ZPR8, mnemonic>; 2899 def _H : sme2_luti4_vector_index<0b01, ZPR16, mnemonic>; 2900 def _S : sme2_luti4_vector_index<0b10, ZPR32, mnemonic>; 2901} 2902 2903// SME2 lookup table expand two contiguous registers 2904class sme2_luti_vector_vg2_index<bits<2> sz, bits<6> opc, RegisterOperand vector_ty, 2905 AsmVectorIndexOpnd index_ty, string mnemonic> 2906 : I<(outs vector_ty:$Zd), 2907 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 2908 mnemonic, "\t$Zd, $ZTt, $Zn$i", 2909 "", []>, Sched<[]> { 2910 bits<5> Zn; 2911 bits<4> Zd; 2912 let Inst{31-19} = 0b1100000010001; 2913 let Inst{18-15} = opc{5-2}; 2914 let Inst{14} = 0b1; 2915 let Inst{13-12} = sz; 2916 let Inst{11-10} = opc{1-0}; 2917 let Inst{9-5} = Zn; 2918 let Inst{4-1} = Zd; 2919 let Inst{0} = 0b0; 2920} 2921 2922class sme2_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 2923 string mnemonic> 2924 : sme2_luti_vector_vg2_index<sz, {1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { 2925 bits<3> i; 2926 let Inst{17-15} = i; 2927} 2928 2929multiclass sme2_luti2_vector_vg2_index<string mnemonic> { 2930 def _B : sme2_luti2_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; 2931 def _H : sme2_luti2_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; 2932 def _S : sme2_luti2_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; 2933} 2934 2935class sme2_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 2936 string mnemonic> 2937 : sme2_luti_vector_vg2_index<sz, {0,1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { 2938 bits<2> i; 2939 let Inst{16-15} = i; 2940} 2941 2942multiclass sme2_luti4_vector_vg2_index<string mnemonic> { 2943 def _B : sme2_luti4_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; 2944 def _H : sme2_luti4_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; 2945 def _S : sme2_luti4_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; 2946} 2947 2948// SME2 lookup table expand four contiguous registers 2949class sme2_luti_vector_vg4_index<bits<2> sz, bits<5>opc, RegisterOperand vector_ty, 2950 AsmVectorIndexOpnd index_ty, string mnemonic> 2951 : I<(outs vector_ty:$Zd), 2952 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 2953 mnemonic, "\t$Zd, $ZTt, $Zn$i", 2954 "", []>, Sched<[]> { 2955 bits<5> Zn; 2956 bits<3> Zd; 2957 let Inst{31-19} = 0b1100000010001; 2958 let Inst{18-16} = opc{4-2}; 2959 let Inst{15-14} = 0b10; 2960 let Inst{13-12} = sz; 2961 let Inst{11-10} = opc{1-0}; 2962 let Inst{9-5} = Zn; 2963 let Inst{4-2} = Zd; 2964 let Inst{1-0} = 0b00; 2965} 2966 2967class sme2_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 2968 string mnemonic> 2969 : sme2_luti_vector_vg4_index<sz, {1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { 2970 bits<2> i; 2971 let Inst{17-16} = i; 2972} 2973 2974multiclass sme2_luti2_vector_vg4_index<string mnemonic> { 2975 def _B : sme2_luti2_vector_vg4_index<0b00, ZZZZ_b_mul_r, mnemonic>; 2976 def _H : sme2_luti2_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; 2977 def _S : sme2_luti2_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; 2978} 2979 2980class sme2_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 2981 string mnemonic> 2982 : sme2_luti_vector_vg4_index<sz, {0,1,?,0,0}, vector_ty, VectorIndexD, mnemonic> { 2983 bits<1> i; 2984 let Inst{16} = i; 2985} 2986 2987multiclass sme2_luti4_vector_vg4_index<string mnemonic> { 2988 def _H : sme2_luti4_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; 2989 def _S : sme2_luti4_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; 2990} 2991 2992//===----------------------------------------------------------------------===// 2993// SME2 MOV 2994class sme2_mova_vec_to_tile_vg2_multi_base<bits<2> sz, bit v, 2995 RegisterOperand tile_ty, 2996 Operand index_ty, 2997 RegisterOperand vector_ty, 2998 string mnemonic> 2999 : I<(outs tile_ty:$ZAd), 3000 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, vector_ty:$Zn), 3001 mnemonic, "\t$ZAd[$Rs, $imm], $Zn", 3002 "", []>, Sched<[]> { 3003 bits<2> Rs; 3004 bits<4> Zn; 3005 let Inst{31-24} = 0b11000000; 3006 let Inst{23-22} = sz; 3007 let Inst{21-16} = 0b000100; 3008 let Inst{15} = v; 3009 let Inst{14-13} = Rs; 3010 let Inst{12-10} = 0b000; 3011 let Inst{9-6} = Zn; 3012 let Inst{5-3} = 0b000; 3013 3014 let Constraints = "$ZAd = $_ZAd"; 3015} 3016 3017multiclass sme2_mova_vec_to_tile_or_array_aliases<int prefer, Instruction inst, 3018 RegisterOperand tile_or_array_ty, 3019 RegisterOperand rv_ty, 3020 Operand index_ty, 3021 RegisterOperand vector_ty, 3022 string mnemonic, 3023 string vg_acronym=""> { 3024 def : InstAlias<mnemonic # "\t$ZAd[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn", 3025 (inst tile_or_array_ty:$ZAd, rv_ty:$Rs, index_ty:$imm, vector_ty:$Zn), prefer>; 3026 3027} 3028 3029// SME2 move vector to tile, two registers 3030multiclass sme2_mova_vec_to_tile_vg2_multi_base<bit v, string mnemonic> { 3031 3032 def _B : sme2_mova_vec_to_tile_vg2_multi_base<0b00, v, 3033 !if(v, TileVectorOpV8, 3034 TileVectorOpH8), 3035 uimm3s2range, ZZ_b_mul_r, 3036 mnemonic> { 3037 bits<3> imm; 3038 let Inst{2-0} = imm; 3039 } 3040 3041 def _H : sme2_mova_vec_to_tile_vg2_multi_base<0b01, v, 3042 !if(v, TileVectorOpV16, 3043 TileVectorOpH16), 3044 uimm2s2range, ZZ_h_mul_r, 3045 mnemonic> { 3046 bits<1> ZAd; 3047 bits<2> imm; 3048 let Inst{2} = ZAd; 3049 let Inst{1-0} = imm; 3050 } 3051 3052 def _S : sme2_mova_vec_to_tile_vg2_multi_base<0b10, v, 3053 !if(v, TileVectorOpV32, 3054 TileVectorOpH32), 3055 uimm1s2range, ZZ_s_mul_r, 3056 mnemonic> { 3057 bits<2> ZAd; 3058 bits<1> imm; 3059 let Inst{2-1} = ZAd; 3060 let Inst{0} = imm; 3061 } 3062 3063 def _D : sme2_mova_vec_to_tile_vg2_multi_base<0b11, v, 3064 !if(v, TileVectorOpV64, 3065 TileVectorOpH64), 3066 uimm0s2range, ZZ_d_mul_r, 3067 mnemonic> { 3068 bits<3> ZAd; 3069 let Inst{2-0} = ZAd; 3070 } 3071 3072 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), 3073 !if(v, TileVectorOpV8, 3074 TileVectorOpH8), 3075 MatrixIndexGPR32Op12_15, 3076 uimm3s2range, ZZ_b_mul_r, 3077 "mov">; 3078 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), 3079 !if(v, TileVectorOpV16, 3080 TileVectorOpH16), 3081 MatrixIndexGPR32Op12_15, 3082 uimm2s2range, ZZ_h_mul_r, 3083 "mov">; 3084 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), 3085 !if(v, TileVectorOpV32, 3086 TileVectorOpH32), 3087 MatrixIndexGPR32Op12_15, 3088 uimm1s2range, ZZ_s_mul_r, 3089 "mov">; 3090 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), 3091 !if(v, TileVectorOpV64, 3092 TileVectorOpH64), 3093 MatrixIndexGPR32Op12_15, 3094 uimm0s2range, ZZ_d_mul_r, 3095 "mov">; 3096 3097 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3098 !if(v, TileVectorOpV8, 3099 TileVectorOpH8), 3100 MatrixIndexGPR32Op12_15, 3101 uimm3s2range, ZZ_b_mul_r, 3102 "mova">; 3103 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3104 !if(v, TileVectorOpV16, 3105 TileVectorOpH16), 3106 MatrixIndexGPR32Op12_15, 3107 uimm2s2range, ZZ_h_mul_r, 3108 "mova">; 3109 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3110 !if(v, TileVectorOpV32, 3111 TileVectorOpH32), 3112 MatrixIndexGPR32Op12_15, 3113 uimm1s2range, ZZ_s_mul_r, 3114 "mova">; 3115 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3116 !if(v, TileVectorOpV64, 3117 TileVectorOpH64), 3118 MatrixIndexGPR32Op12_15, 3119 uimm0s2range, ZZ_d_mul_r, 3120 "mova">; 3121 3122 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3123 !if(v, TileVectorOpV8, 3124 TileVectorOpH8), 3125 MatrixIndexGPR32Op12_15, 3126 uimm3s2range, ZZ_b_mul_r, 3127 "mova">; 3128 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3129 !if(v, TileVectorOpV16, 3130 TileVectorOpH16), 3131 MatrixIndexGPR32Op12_15, 3132 uimm2s2range, ZZ_h_mul_r, 3133 "mova">; 3134 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3135 !if(v, TileVectorOpV32, 3136 TileVectorOpH32), 3137 MatrixIndexGPR32Op12_15, 3138 uimm1s2range, ZZ_s_mul_r, 3139 "mova">; 3140 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3141 !if(v, TileVectorOpV64, 3142 TileVectorOpH64), 3143 MatrixIndexGPR32Op12_15, 3144 uimm0s2range, ZZ_d_mul_r, 3145 "mova">; 3146} 3147 3148multiclass sme2_mova_vec_to_tile_vg2_multi<string mnemonic>{ 3149 defm _H : sme2_mova_vec_to_tile_vg2_multi_base<0b0, mnemonic>; 3150 defm _V : sme2_mova_vec_to_tile_vg2_multi_base<0b1, mnemonic>; 3151} 3152 3153class sme2_mova_vec_to_tile_vg4_multi_base<bits<2> sz, bit v, bits<3> op, 3154 RegisterOperand tile_ty, 3155 Operand index_ty, 3156 RegisterOperand vector_ty, 3157 string mnemonic> 3158 : I<(outs tile_ty:$ZAd), 3159 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, 3160 vector_ty:$Zn), 3161 mnemonic, 3162 "\t$ZAd[$Rs, $imm], $Zn", 3163 "", []>, Sched<[]> { 3164 bits<2> Rs; 3165 bits<3> Zn; 3166 let Inst{31-24} = 0b11000000; 3167 let Inst{23-22} = sz; 3168 let Inst{21-16} = 0b000100; 3169 let Inst{15} = v; 3170 let Inst{14-13} = Rs; 3171 let Inst{12-10} = 0b001; 3172 let Inst{9-7} = Zn; 3173 let Inst{6-3} = 0b0000; 3174 let Inst{2-0} = op; 3175 let Constraints = "$ZAd = $_ZAd"; 3176} 3177 3178// SME2 move vector to tile, four registers 3179multiclass sme2_mova_vec_to_tile_vg4_multi_base<bit v, string mnemonic> { 3180 3181 def _B : sme2_mova_vec_to_tile_vg4_multi_base<0b00, v, {0,?,?}, 3182 !if(v, TileVectorOpV8, 3183 TileVectorOpH8), 3184 uimm2s4range, ZZZZ_b_mul_r, 3185 mnemonic> { 3186 bits<2> imm; 3187 let Inst{1-0} = imm; 3188 } 3189 3190 def _H : sme2_mova_vec_to_tile_vg4_multi_base<0b01, v, {0,?,?}, 3191 !if(v, TileVectorOpV16, 3192 TileVectorOpH16), 3193 uimm1s4range, ZZZZ_h_mul_r, 3194 mnemonic> { 3195 bits<1> ZAd; 3196 bits<1> imm; 3197 let Inst{1} = ZAd; 3198 let Inst{0} = imm; 3199 } 3200 3201 def _S : sme2_mova_vec_to_tile_vg4_multi_base<0b10, v, {0,?,?}, 3202 !if(v, TileVectorOpV32, 3203 TileVectorOpH32), 3204 uimm0s4range, ZZZZ_s_mul_r, 3205 mnemonic> { 3206 bits<2> ZAd; 3207 let Inst{1-0} = ZAd; 3208 } 3209 3210 def _D : sme2_mova_vec_to_tile_vg4_multi_base<0b11, v, {?,?,?}, 3211 !if(v, TileVectorOpV64, 3212 TileVectorOpH64), 3213 uimm0s4range, ZZZZ_d_mul_r, 3214 mnemonic> { 3215 bits<3> ZAd; 3216 let Inst{2-0} = ZAd; 3217 } 3218 3219 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), 3220 !if(v, TileVectorOpV8, 3221 TileVectorOpH8), 3222 MatrixIndexGPR32Op12_15, 3223 uimm2s4range, ZZZZ_b_mul_r, 3224 "mov">; 3225 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), 3226 !if(v, TileVectorOpV16, 3227 TileVectorOpH16), 3228 MatrixIndexGPR32Op12_15, 3229 uimm1s4range, ZZZZ_h_mul_r, 3230 "mov">; 3231 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), 3232 !if(v, TileVectorOpV32, 3233 TileVectorOpH32), 3234 MatrixIndexGPR32Op12_15, 3235 uimm0s4range, ZZZZ_s_mul_r, 3236 "mov">; 3237 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), 3238 !if(v, TileVectorOpV64, 3239 TileVectorOpH64), 3240 MatrixIndexGPR32Op12_15, 3241 uimm0s4range, ZZZZ_d_mul_r, 3242 "mov">; 3243 3244 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3245 !if(v, TileVectorOpV8, 3246 TileVectorOpH8), 3247 MatrixIndexGPR32Op12_15, 3248 uimm2s4range, ZZZZ_b_mul_r, 3249 "mova">; 3250 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3251 !if(v, TileVectorOpV16, 3252 TileVectorOpH16), 3253 MatrixIndexGPR32Op12_15, 3254 uimm1s4range, ZZZZ_h_mul_r, 3255 "mova">; 3256 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3257 !if(v, TileVectorOpV32, 3258 TileVectorOpH32), 3259 MatrixIndexGPR32Op12_15, 3260 uimm0s4range, ZZZZ_s_mul_r, 3261 "mova">; 3262 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3263 !if(v, TileVectorOpV64, 3264 TileVectorOpH64), 3265 MatrixIndexGPR32Op12_15, 3266 uimm0s4range, ZZZZ_d_mul_r, 3267 "mova">; 3268 3269} 3270 3271multiclass sme2_mova_vec_to_tile_vg4_multi<string mnemonic>{ 3272 defm _H : sme2_mova_vec_to_tile_vg4_multi_base<0b0, mnemonic>; 3273 defm _V : sme2_mova_vec_to_tile_vg4_multi_base<0b1, mnemonic>; 3274} 3275 3276// SME Move into Array 3277class sme2_mova_vec_to_array_vg24_multi< bits<5> op, RegisterOperand array_ty, 3278 RegisterOperand vector_ty, 3279 string mnemonic, 3280 string vg_acronym=""> 3281 : I<(outs array_ty:$ZAd), 3282 (ins array_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm, 3283 vector_ty:$Zn), 3284 mnemonic, "\t$ZAd[$Rs, $imm, " # vg_acronym # "], $Zn", 3285 "", []>, Sched<[]> { 3286 bits<2> Rs; 3287 bits<3> imm; 3288 let Inst{31-15} = 0b11000000000001000; 3289 let Inst{14-13} = Rs; 3290 let Inst{12-11} = 0b01; 3291 let Inst{10-6} = op; 3292 let Inst{5-3} = 0b000; 3293 let Inst{2-0} = imm; 3294 3295 let Constraints = "$ZAd = $_ZAd"; 3296} 3297 3298// MOVA (vector to array, two registers) 3299multiclass sme2_mova_vec_to_array_vg2_multi<string mnemonic> { 3300 def NAME : sme2_mova_vec_to_array_vg24_multi<{0,?,?,?,?}, MatrixOp64, 3301 ZZ_d_mul_r, mnemonic, "vgx2">{ 3302 bits<4> Zn; 3303 let Inst{9-6} = Zn; 3304 } 3305 3306 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3307 MatrixOp8, 3308 MatrixIndexGPR32Op8_11, 3309 sme_elm_idx0_7, ZZ_b_mul_r, 3310 "mova">; 3311 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3312 MatrixOp16, 3313 MatrixIndexGPR32Op8_11, 3314 sme_elm_idx0_7, ZZ_h_mul_r, 3315 "mova">; 3316 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3317 MatrixOp32, 3318 MatrixIndexGPR32Op8_11, 3319 sme_elm_idx0_7, ZZ_s_mul_r, 3320 "mova">; 3321 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3322 MatrixOp64, 3323 MatrixIndexGPR32Op8_11, 3324 sme_elm_idx0_7, ZZ_d_mul_r, 3325 "mova">; 3326 3327 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3328 MatrixOp8, 3329 MatrixIndexGPR32Op8_11, 3330 sme_elm_idx0_7, ZZ_b_mul_r, 3331 "mova", "vgx2">; 3332 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3333 MatrixOp16, 3334 MatrixIndexGPR32Op8_11, 3335 sme_elm_idx0_7, ZZ_h_mul_r, 3336 "mova", "vgx2">; 3337 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3338 MatrixOp32, 3339 MatrixIndexGPR32Op8_11, 3340 sme_elm_idx0_7, ZZ_s_mul_r, 3341 "mova", "vgx2">; 3342 3343 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3344 MatrixOp8, 3345 MatrixIndexGPR32Op8_11, 3346 sme_elm_idx0_7, ZZ_b_mul_r, 3347 "mov">; 3348 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3349 MatrixOp16, 3350 MatrixIndexGPR32Op8_11, 3351 sme_elm_idx0_7, ZZ_h_mul_r, 3352 "mov">; 3353 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3354 MatrixOp32, 3355 MatrixIndexGPR32Op8_11, 3356 sme_elm_idx0_7, ZZ_s_mul_r, 3357 "mov">; 3358 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3359 MatrixOp64, 3360 MatrixIndexGPR32Op8_11, 3361 sme_elm_idx0_7, ZZ_d_mul_r, 3362 "mov">; 3363 3364 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3365 MatrixOp8, 3366 MatrixIndexGPR32Op8_11, 3367 sme_elm_idx0_7, ZZ_b_mul_r, 3368 "mov", "vgx2">; 3369 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3370 MatrixOp16, 3371 MatrixIndexGPR32Op8_11, 3372 sme_elm_idx0_7, ZZ_h_mul_r, 3373 "mov", "vgx2">; 3374 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3375 MatrixOp32, 3376 MatrixIndexGPR32Op8_11, 3377 sme_elm_idx0_7, ZZ_s_mul_r, 3378 "mov", "vgx2">; 3379 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), 3380 MatrixOp64, 3381 MatrixIndexGPR32Op8_11, 3382 sme_elm_idx0_7, ZZ_d_mul_r, 3383 "mov", "vgx2">; 3384} 3385 3386// MOVA (vector to array, four registers) 3387multiclass sme2_mova_vec_to_array_vg4_multi<string mnemonic> { 3388 def NAME : sme2_mova_vec_to_array_vg24_multi<{1,?,?,?,0}, MatrixOp64, 3389 ZZZZ_d_mul_r, mnemonic, "vgx4"> { 3390 bits<3> Zn; 3391 let Inst{9-7} = Zn; 3392 } 3393 3394 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3395 MatrixOp8, 3396 MatrixIndexGPR32Op8_11, 3397 sme_elm_idx0_7, ZZZZ_b_mul_r, 3398 "mova">; 3399 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3400 MatrixOp16, 3401 MatrixIndexGPR32Op8_11, 3402 sme_elm_idx0_7, ZZZZ_h_mul_r, 3403 "mova">; 3404 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3405 MatrixOp32, 3406 MatrixIndexGPR32Op8_11, 3407 sme_elm_idx0_7, ZZZZ_s_mul_r, 3408 "mova">; 3409 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3410 MatrixOp64, 3411 MatrixIndexGPR32Op8_11, 3412 sme_elm_idx0_7, ZZZZ_d_mul_r, 3413 "mova">; 3414 3415 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3416 MatrixOp8, 3417 MatrixIndexGPR32Op8_11, 3418 sme_elm_idx0_7, ZZZZ_b_mul_r, 3419 "mova", "vgx4">; 3420 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3421 MatrixOp16, 3422 MatrixIndexGPR32Op8_11, 3423 sme_elm_idx0_7, ZZZZ_h_mul_r, 3424 "mova", "vgx4">; 3425 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3426 MatrixOp32, 3427 MatrixIndexGPR32Op8_11, 3428 sme_elm_idx0_7, ZZZZ_s_mul_r, 3429 "mova", "vgx4">; 3430 3431 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3432 MatrixOp8, 3433 MatrixIndexGPR32Op8_11, 3434 sme_elm_idx0_7, ZZZZ_b_mul_r, 3435 "mov">; 3436 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3437 MatrixOp16, 3438 MatrixIndexGPR32Op8_11, 3439 sme_elm_idx0_7, ZZZZ_h_mul_r, 3440 "mov">; 3441 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3442 MatrixOp32, 3443 MatrixIndexGPR32Op8_11, 3444 sme_elm_idx0_7, ZZZZ_s_mul_r, 3445 "mov">; 3446 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3447 MatrixOp64, 3448 MatrixIndexGPR32Op8_11, 3449 sme_elm_idx0_7, ZZZZ_d_mul_r, 3450 "mov">; 3451 3452 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3453 MatrixOp8, 3454 MatrixIndexGPR32Op8_11, 3455 sme_elm_idx0_7, ZZZZ_b_mul_r, 3456 "mov", "vgx4">; 3457 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3458 MatrixOp16, 3459 MatrixIndexGPR32Op8_11, 3460 sme_elm_idx0_7, ZZZZ_h_mul_r, 3461 "mov", "vgx4">; 3462 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3463 MatrixOp32, 3464 MatrixIndexGPR32Op8_11, 3465 sme_elm_idx0_7, ZZZZ_s_mul_r, 3466 "mov", "vgx4">; 3467 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), 3468 MatrixOp64, 3469 MatrixIndexGPR32Op8_11, 3470 sme_elm_idx0_7, ZZZZ_d_mul_r, 3471 "mov", "vgx4">; 3472 3473} 3474 3475class sme2_mova_tile_to_vec_vg2_multi_base<bits<2> sz, bit v, bits<3> op, 3476 RegisterOperand vector_ty, 3477 RegisterOperand tile_ty, 3478 Operand index_ty, 3479 string mnemonic> 3480 : I<!if(op{1}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), 3481 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 3482 mnemonic, 3483 "\t$Zd, $ZAn[$Rs, $imm]", 3484 "", []>, Sched<[]> { 3485 bits<4> Zd; 3486 bits<2> Rs; 3487 let Inst{31-24} = 0b11000000; 3488 let Inst{23-22} = sz; 3489 let Inst{21-16} = 0b000110; 3490 let Inst{15} = v; 3491 let Inst{14-13} = Rs; 3492 let Inst{12-11} = 0b00; 3493 let Inst{10-8} = op; 3494 let Inst{4-1} = Zd; 3495 let Inst{0} = 0b0; 3496 3497 let Constraints = !if(op{1}, "$ZAn = $_ZAn", ""); 3498} 3499 3500multiclass sme2_mova_tile_or_array_to_vec_aliases<int op, Instruction inst, 3501 RegisterOperand vector_ty, 3502 RegisterOperand tile_or_array_ty, 3503 RegisterOperand rv_ty, 3504 Operand index_ty, 3505 string mnemonic, 3506 string vg_acronym=""> { 3507def : InstAlias<mnemonic # "\t$Zd, $ZAn[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", 3508 (inst vector_ty:$Zd, tile_or_array_ty:$ZAn, rv_ty:$Rs, index_ty:$imm), op>; 3509 3510} 3511 3512multiclass sme2_mova_tile_to_vec_vg2_multi_inst<bit v, bits<3> opc, string mnemonic> { 3513 3514 def _B : sme2_mova_tile_to_vec_vg2_multi_base<0b00, v, opc, ZZ_b_mul_r, 3515 !if(v, TileVectorOpV8, 3516 TileVectorOpH8), 3517 uimm3s2range, mnemonic> { 3518 bits<3> imm; 3519 let Inst{7-5} = imm; 3520 } 3521 3522 def _H : sme2_mova_tile_to_vec_vg2_multi_base<0b01, v, opc, ZZ_h_mul_r, 3523 !if(v, TileVectorOpV16, 3524 TileVectorOpH16), 3525 uimm2s2range, mnemonic> { 3526 bits<1> ZAn; 3527 bits<2> imm; 3528 let Inst{7} = ZAn; 3529 let Inst{6-5} = imm; 3530 } 3531 3532 def _S : sme2_mova_tile_to_vec_vg2_multi_base<0b10, v, opc, ZZ_s_mul_r, 3533 !if(v, TileVectorOpV32, 3534 TileVectorOpH32), 3535 uimm1s2range, mnemonic> { 3536 bits<2> ZAn; 3537 bits<1> imm; 3538 let Inst{7-6} = ZAn; 3539 let Inst{5} = imm; 3540 } 3541 3542 def _D : sme2_mova_tile_to_vec_vg2_multi_base<0b11, v, opc, ZZ_d_mul_r, 3543 !if(v, TileVectorOpV64, 3544 TileVectorOpH64), 3545 uimm0s2range, mnemonic> { 3546 bits<3> ZAn; 3547 let Inst{7-5} = ZAn; 3548 } 3549 3550 if !eq(mnemonic, "mova") then { 3551 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _B), 3552 ZZ_b_mul_r, 3553 !if(v, TileVectorOpV8, 3554 TileVectorOpH8), 3555 MatrixIndexGPR32Op12_15, 3556 uimm3s2range, "mov">; 3557 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _H), 3558 ZZ_h_mul_r, 3559 !if(v, TileVectorOpV16, 3560 TileVectorOpH16), 3561 MatrixIndexGPR32Op12_15, 3562 uimm2s2range, "mov">; 3563 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), 3564 ZZ_s_mul_r, 3565 !if(v, TileVectorOpV32, 3566 TileVectorOpH32), 3567 MatrixIndexGPR32Op12_15, 3568 uimm1s2range, "mov">; 3569 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), 3570 ZZ_d_mul_r, 3571 !if(v, TileVectorOpV64, 3572 TileVectorOpH64), 3573 MatrixIndexGPR32Op12_15, 3574 uimm0s2range, "mov">; 3575 } 3576 3577 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _B), 3578 ZZ_b_mul_r, 3579 !if(v, TileVectorOpV8, 3580 TileVectorOpH8), 3581 MatrixIndexGPR32Op12_15, 3582 uimm3s2range, mnemonic>; 3583 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _H), 3584 ZZ_h_mul_r, 3585 !if(v, TileVectorOpV16, 3586 TileVectorOpH16), 3587 MatrixIndexGPR32Op12_15, 3588 uimm2s2range, mnemonic>; 3589 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), 3590 ZZ_s_mul_r, 3591 !if(v, TileVectorOpV32, 3592 TileVectorOpH32), 3593 MatrixIndexGPR32Op12_15, 3594 uimm1s2range, mnemonic>; 3595 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), 3596 ZZ_d_mul_r, 3597 !if(v, TileVectorOpV64, 3598 TileVectorOpH64), 3599 MatrixIndexGPR32Op12_15, 3600 uimm0s2range, mnemonic>; 3601 3602} 3603 3604// SME2 move tile to vector, two registers 3605multiclass sme2_mova_tile_to_vec_vg2_multi<string mnemonic>{ 3606 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b000, mnemonic>; 3607 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b000, mnemonic>; 3608} 3609 3610// SME2p1 move tile to vector and zero tile, two registers 3611multiclass sme2p1_movaz_tile_to_vec_vg2<string mnemonic>{ 3612 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b010, mnemonic>; 3613 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b010, mnemonic>; 3614} 3615 3616class sme2_mova_tile_to_vec_vg4_multi_base<bits<2> sz, bit v, bits<6> op, 3617 RegisterOperand vector_ty, 3618 RegisterOperand tile_ty, 3619 Operand index_ty, 3620 string mnemonic> 3621 : I<!if(op{4}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), 3622 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 3623 mnemonic, 3624 "\t$Zd, $ZAn[$Rs, $imm]", 3625 "", []>, Sched<[]> { 3626 bits<3> Zd; 3627 bits<2> Rs; 3628 let Inst{31-24} = 0b11000000; 3629 let Inst{23-22} = sz; 3630 let Inst{21-16} = 0b000110; 3631 let Inst{15} = v; 3632 let Inst{14-13} = Rs; 3633 let Inst{12-11} = 0b00; 3634 let Inst{10-5} = op{5-0}; 3635 let Inst{4-2} = Zd; 3636 let Inst{1-0} = 0b00; 3637 3638 let Constraints = !if(op{4}, "$ZAn = $_ZAn", ""); 3639} 3640 3641multiclass sme2_mova_tile_to_vec_vg4_multi_base<bit v, bits<3> opc, string mnemonic> { 3642 3643 def _B : sme2_mova_tile_to_vec_vg4_multi_base<0b00, v, {opc,0,?,?}, 3644 ZZZZ_b_mul_r, 3645 !if(v, TileVectorOpV8, 3646 TileVectorOpH8), 3647 uimm2s4range, mnemonic> { 3648 bits<2> imm; 3649 let Inst{6-5} = imm; 3650 } 3651 3652 def _H : sme2_mova_tile_to_vec_vg4_multi_base<0b01, v, {opc,0,?,?}, 3653 ZZZZ_h_mul_r, 3654 !if(v, TileVectorOpV16, 3655 TileVectorOpH16), 3656 uimm1s4range, mnemonic> { 3657 bits<1> ZAn; 3658 bits<1> imm; 3659 let Inst{6} = ZAn; 3660 let Inst{5} = imm; 3661 } 3662 3663 def _S : sme2_mova_tile_to_vec_vg4_multi_base<0b10, v, {opc,0,?,?}, 3664 ZZZZ_s_mul_r, 3665 !if(v, TileVectorOpV32, 3666 TileVectorOpH32), 3667 uimm0s4range, mnemonic> { 3668 bits<2> ZAn; 3669 let Inst{6-5} = ZAn; 3670 } 3671 3672 def _D : sme2_mova_tile_to_vec_vg4_multi_base<0b11, v, {opc,?,?,?}, 3673 ZZZZ_d_mul_r, 3674 !if(v, TileVectorOpV64, 3675 TileVectorOpH64), 3676 uimm0s4range, mnemonic> { 3677 bits<3> ZAn; 3678 let Inst{7-5} = ZAn; 3679 } 3680 3681 if !eq(mnemonic, "mova") then { 3682 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _B), 3683 ZZZZ_b_mul_r, 3684 !if(v, TileVectorOpV8, 3685 TileVectorOpH8), 3686 MatrixIndexGPR32Op12_15, 3687 uimm2s4range, "mov">; 3688 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _H), 3689 ZZZZ_h_mul_r, 3690 !if(v, TileVectorOpV16, 3691 TileVectorOpH16), 3692 MatrixIndexGPR32Op12_15, 3693 uimm1s4range, "mov">; 3694 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), 3695 ZZZZ_s_mul_r, 3696 !if(v, TileVectorOpV32, 3697 TileVectorOpH32), 3698 MatrixIndexGPR32Op12_15, 3699 uimm0s4range, "mov">; 3700 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), 3701 ZZZZ_d_mul_r, 3702 !if(v, TileVectorOpV64, 3703 TileVectorOpH64), 3704 MatrixIndexGPR32Op12_15, 3705 uimm0s4range, "mov">; 3706 } 3707 3708 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _B), 3709 ZZZZ_b_mul_r, 3710 !if(v, TileVectorOpV8, 3711 TileVectorOpH8), 3712 MatrixIndexGPR32Op12_15, 3713 uimm2s4range, mnemonic>; 3714 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _H), 3715 ZZZZ_h_mul_r, 3716 !if(v, TileVectorOpV16, 3717 TileVectorOpH16), 3718 MatrixIndexGPR32Op12_15, 3719 uimm1s4range, mnemonic>; 3720 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), 3721 ZZZZ_s_mul_r, 3722 !if(v, TileVectorOpV32, 3723 TileVectorOpH32), 3724 MatrixIndexGPR32Op12_15, 3725 uimm0s4range, mnemonic>; 3726 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), 3727 ZZZZ_d_mul_r, 3728 !if(v, TileVectorOpV64, 3729 TileVectorOpH64), 3730 MatrixIndexGPR32Op12_15, 3731 uimm0s4range, mnemonic>; 3732 3733} 3734 3735// SME2 move tile to vector, four registers 3736multiclass sme2_mova_tile_to_vec_vg4_multi<string mnemonic>{ 3737 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b100, mnemonic>; 3738 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b100, mnemonic>; 3739} 3740 3741// SME2p1 move tile to vector and zero tile, four registers 3742multiclass sme2p1_movaz_tile_to_vec_vg4<string mnemonic>{ 3743 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b110, mnemonic>; 3744 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b110, mnemonic>; 3745} 3746 3747 3748class sme2_mova_array_to_vec_vg24_multi<bits<4>op, RegisterOperand vector_ty, 3749 RegisterOperand array_ty, 3750 string mnemonic, string vg_acronym> 3751 : I<!if(op{2}, (outs vector_ty:$Zd, array_ty:$_ZAn), (outs vector_ty:$Zd)), 3752 (ins array_ty:$ZAn, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm), 3753 mnemonic, 3754 "\t$Zd, $ZAn[$Rs, $imm, " # vg_acronym # "]", 3755 "", []>, Sched<[]> { 3756 bits<2> Rs; 3757 bits<3> imm; 3758 let Inst{31-15} = 0b11000000000001100; 3759 let Inst{14-13} = Rs; 3760 let Inst{12-11} = 0b01; 3761 let Inst{10-8} = op{3-1}; 3762 let Inst{7-5} = imm; 3763 let Inst{1} = op{0}; 3764 let Inst{0} = 0b0; 3765 let Constraints = !if(op{2}, "$ZAn = $_ZAn", ""); 3766} 3767 3768// move array to vector, two registers. 3769multiclass sme2_mova_array_to_vec_vg2_multi<bits<3> opc, string mnemonic> { 3770 def NAME : sme2_mova_array_to_vec_vg24_multi<{opc,?}, ZZ_d_mul_r, MatrixOp64, 3771 mnemonic, "vgx2"> { 3772 bits<4> Zd; 3773 let Inst{4-1} = Zd; 3774 } 3775 3776 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3777 ZZ_b_mul_r, MatrixOp8, 3778 MatrixIndexGPR32Op8_11, 3779 sme_elm_idx0_7, mnemonic>; 3780 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3781 ZZ_h_mul_r, MatrixOp16, 3782 MatrixIndexGPR32Op8_11, 3783 sme_elm_idx0_7, mnemonic>; 3784 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3785 ZZ_s_mul_r, MatrixOp32, 3786 MatrixIndexGPR32Op8_11, 3787 sme_elm_idx0_7, mnemonic>; 3788 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3789 ZZ_d_mul_r, MatrixOp64, 3790 MatrixIndexGPR32Op8_11, 3791 sme_elm_idx0_7, mnemonic>; 3792 3793 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3794 ZZ_b_mul_r, MatrixOp8, 3795 MatrixIndexGPR32Op8_11, 3796 sme_elm_idx0_7, mnemonic, "vgx2">; 3797 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3798 ZZ_h_mul_r, MatrixOp16, 3799 MatrixIndexGPR32Op8_11, 3800 sme_elm_idx0_7, mnemonic, "vgx2">; 3801 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3802 ZZ_s_mul_r, MatrixOp32, 3803 MatrixIndexGPR32Op8_11, 3804 sme_elm_idx0_7, mnemonic, "vgx2">; 3805 3806 if !eq(mnemonic, "mova") then { 3807 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3808 ZZ_b_mul_r, MatrixOp8, 3809 MatrixIndexGPR32Op8_11, 3810 sme_elm_idx0_7, "mov">; 3811 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3812 ZZ_h_mul_r, MatrixOp16, 3813 MatrixIndexGPR32Op8_11, 3814 sme_elm_idx0_7, "mov">; 3815 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3816 ZZ_s_mul_r, MatrixOp32, 3817 MatrixIndexGPR32Op8_11, 3818 sme_elm_idx0_7, "mov">; 3819 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3820 ZZ_d_mul_r, MatrixOp64, 3821 MatrixIndexGPR32Op8_11, 3822 sme_elm_idx0_7, "mov">; 3823 3824 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3825 ZZ_b_mul_r, MatrixOp8, 3826 MatrixIndexGPR32Op8_11, 3827 sme_elm_idx0_7, "mov", "vgx2">; 3828 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3829 ZZ_h_mul_r, MatrixOp16, 3830 MatrixIndexGPR32Op8_11, 3831 sme_elm_idx0_7, "mov", "vgx2">; 3832 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3833 ZZ_s_mul_r, MatrixOp32, 3834 MatrixIndexGPR32Op8_11, 3835 sme_elm_idx0_7, "mov", "vgx2">; 3836 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), 3837 ZZ_d_mul_r, MatrixOp64, 3838 MatrixIndexGPR32Op8_11, 3839 sme_elm_idx0_7, "mov", "vgx2">; 3840 } 3841} 3842 3843// move array to vector, four registers 3844multiclass sme2_mova_array_to_vec_vg4_multi<bits<4> opc, string mnemonic> { 3845 def NAME : sme2_mova_array_to_vec_vg24_multi<opc, ZZZZ_d_mul_r, MatrixOp64, 3846 mnemonic, "vgx4"> { 3847 bits<3> Zd; 3848 let Inst{4-2} = Zd; 3849 } 3850 3851 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3852 ZZZZ_b_mul_r, MatrixOp8, 3853 MatrixIndexGPR32Op8_11, 3854 sme_elm_idx0_7, mnemonic>; 3855 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3856 ZZZZ_h_mul_r, MatrixOp16, 3857 MatrixIndexGPR32Op8_11, 3858 sme_elm_idx0_7, mnemonic>; 3859 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3860 ZZZZ_s_mul_r, MatrixOp32, 3861 MatrixIndexGPR32Op8_11, 3862 sme_elm_idx0_7, mnemonic>; 3863 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3864 ZZZZ_d_mul_r, MatrixOp64, 3865 MatrixIndexGPR32Op8_11, 3866 sme_elm_idx0_7, mnemonic>; 3867 3868 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3869 ZZZZ_b_mul_r, MatrixOp8, 3870 MatrixIndexGPR32Op8_11, 3871 sme_elm_idx0_7, mnemonic, "vgx4">; 3872 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3873 ZZZZ_h_mul_r, MatrixOp16, 3874 MatrixIndexGPR32Op8_11, 3875 sme_elm_idx0_7, mnemonic, "vgx4">; 3876 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3877 ZZZZ_s_mul_r, MatrixOp32, 3878 MatrixIndexGPR32Op8_11, 3879 sme_elm_idx0_7, mnemonic, "vgx4">; 3880 3881 if !eq(mnemonic, "mova") then { 3882 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3883 ZZZZ_b_mul_r, MatrixOp8, 3884 MatrixIndexGPR32Op8_11, 3885 sme_elm_idx0_7, "mov">; 3886 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3887 ZZZZ_h_mul_r, MatrixOp16, 3888 MatrixIndexGPR32Op8_11, 3889 sme_elm_idx0_7, "mov">; 3890 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3891 ZZZZ_s_mul_r, MatrixOp32, 3892 MatrixIndexGPR32Op8_11, 3893 sme_elm_idx0_7, "mov">; 3894 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3895 ZZZZ_d_mul_r, MatrixOp64, 3896 MatrixIndexGPR32Op8_11, 3897 sme_elm_idx0_7, "mov">; 3898 3899 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3900 ZZZZ_b_mul_r, MatrixOp8, 3901 MatrixIndexGPR32Op8_11, 3902 sme_elm_idx0_7, "mov", "vgx4">; 3903 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3904 ZZZZ_h_mul_r, MatrixOp16, 3905 MatrixIndexGPR32Op8_11, 3906 sme_elm_idx0_7, "mov", "vgx4">; 3907 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3908 ZZZZ_s_mul_r, MatrixOp32, 3909 MatrixIndexGPR32Op8_11, 3910 sme_elm_idx0_7, "mov", "vgx4">; 3911 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), 3912 ZZZZ_d_mul_r, MatrixOp64, 3913 MatrixIndexGPR32Op8_11, 3914 sme_elm_idx0_7, "mov", "vgx4">; 3915 } 3916} 3917 3918//===----------------------------------------------------------------------===// 3919// SME2 multi-vec saturating shift right narrow 3920class sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> 3921 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, vecshiftR16:$imm4), 3922 mnemonic, "\t$Zd, $Zn, $imm4", 3923 "", []>, Sched<[]> { 3924 bits<4> imm4; 3925 bits<4> Zn; 3926 bits<5> Zd; 3927 let Inst{31-21} = 0b11000001111; 3928 let Inst{20} = op; 3929 let Inst{19-16} = imm4; 3930 let Inst{15-10} = 0b110101; 3931 let Inst{9-6} = Zn; 3932 let Inst{5} = u; 3933 let Inst{4-0} = Zd; 3934} 3935 3936multiclass sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> { 3937 def _H : sme2_sat_shift_vector_vg2<mnemonic, op, u>; 3938} 3939 3940class sme2_sat_shift_vector_vg4<bits<2> sz, bits<3> op, ZPRRegOp zpr_ty, 3941 RegisterOperand vector_ty, Operand imm_ty, 3942 string mnemonic> 3943 : I<(outs zpr_ty:$Zd), (ins vector_ty:$Zn, imm_ty:$imm), 3944 mnemonic, "\t$Zd, $Zn, $imm", 3945 "", []>, Sched<[]> { 3946 bits<3> Zn; 3947 bits<5> Zd; 3948 let Inst{31-24} = 0b11000001; 3949 let Inst{23-22} = sz; 3950 let Inst{21} = 0b1; 3951 // Inst{20-16} = imm5; 3952 let Inst{15-11} = 0b11011; 3953 let Inst{10} = op{2}; 3954 let Inst{9-7} = Zn; 3955 let Inst{6-5} = op{1-0}; 3956 let Inst{4-0} = Zd; 3957} 3958 3959multiclass sme2_sat_shift_vector_vg4<string mnemonic, bits<3> op> { 3960 def _B : sme2_sat_shift_vector_vg4<{0,1}, op, ZPR8, ZZZZ_s_mul_r, vecshiftR32, 3961 mnemonic>{ 3962 bits<5> imm; 3963 let Inst{20-16} = imm; 3964 } 3965 def _H : sme2_sat_shift_vector_vg4<{1,?}, op, ZPR16, ZZZZ_d_mul_r, vecshiftR64, 3966 mnemonic> { 3967 bits<6> imm; 3968 let Inst{22} = imm{5}; 3969 let Inst{20-16} = imm{4-0}; 3970 } 3971} 3972 3973//===----------------------------------------------------------------------===// 3974// SME2 Multi-vector - SVE Select 3975class sme2_sel_vector_vg24<bits<2> sz, bits<4> op, RegisterOperand vector_ty, 3976 string mnemonic> 3977 : I<(outs vector_ty:$Zd), 3978 (ins PNRAny_p8to15:$PNg, vector_ty:$Zn, vector_ty:$Zm), 3979 mnemonic, "\t$Zd, $PNg, $Zn, $Zm", 3980 "", []>, Sched<[]> { 3981 bits<3> PNg; 3982 let Inst{31-24} = 0b11000001; 3983 let Inst{23-22} = sz; 3984 let Inst{21} = 0b1; 3985 let Inst{17-16} = op{3-2}; 3986 let Inst{15-13} = 0b100; 3987 let Inst{12-10} = PNg; 3988 let Inst{6} = op{1}; 3989 let Inst{5} = 0b0; 3990 let Inst{1} = op{0}; 3991 let Inst{0} = 0b0; 3992} 3993 3994class sme2_sel_vector_vg2<bits<2> sz, RegisterOperand vector_ty, 3995 string mnemonic> 3996 : sme2_sel_vector_vg24<sz, {?,0,?,?}, vector_ty, mnemonic> { 3997 bits<4> Zm; 3998 bits<4> Zn; 3999 bits<4> Zd; 4000 let Inst{20-17} = Zm; 4001 let Inst{9-6} = Zn; 4002 let Inst{4-1} = Zd; 4003} 4004 4005multiclass sme2_sel_vector_vg2<string mnemonic>{ 4006 def _B : sme2_sel_vector_vg2<0b00, ZZ_b_mul_r, mnemonic>; 4007 def _H : sme2_sel_vector_vg2<0b01, ZZ_h_mul_r, mnemonic>; 4008 def _S : sme2_sel_vector_vg2<0b10, ZZ_s_mul_r, mnemonic>; 4009 def _D : sme2_sel_vector_vg2<0b11, ZZ_d_mul_r, mnemonic>; 4010} 4011class sme2_sel_vector_vg4<bits<2> sz, RegisterOperand vector_ty, 4012 string mnemonic> 4013 : sme2_sel_vector_vg24<sz, 0b0100, vector_ty, mnemonic> { 4014 bits<3> Zm; 4015 bits<3> Zn; 4016 bits<3> Zd; 4017 let Inst{20-18} = Zm; 4018 let Inst{9-7} = Zn; 4019 let Inst{4-2} = Zd; 4020} 4021multiclass sme2_sel_vector_vg4<string mnemonic> { 4022 def _B : sme2_sel_vector_vg4<0b00, ZZZZ_b_mul_r, mnemonic>; 4023 def _H : sme2_sel_vector_vg4<0b01, ZZZZ_h_mul_r, mnemonic>; 4024 def _S : sme2_sel_vector_vg4<0b10, ZZZZ_s_mul_r, mnemonic>; 4025 def _D : sme2_sel_vector_vg4<0b11, ZZZZ_d_mul_r, mnemonic>; 4026} 4027 4028//===----------------------------------------------------------------------===// 4029// Non contiguous Load and Store 4030 4031class sme2_ld_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, 4032 RegisterOperand multi_vector_ty, 4033 RegisterOperand gpr_ty, 4034 string mnemonic> 4035 : I<(outs multi_vector_ty:$Zt), 4036 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4037 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 4038 "", []>, Sched<[]> { 4039 bits<5> Rm; 4040 bits<3> PNg; 4041 bits<5> Rn; 4042 bits<4> Zt; 4043 let Inst{31-21} = 0b10100001000; 4044 let Inst{20-16} = Rm; 4045 let Inst{15} = 0b0; 4046 let Inst{14-13} = msz; 4047 let Inst{12-10} = PNg; 4048 let Inst{9-5} = Rn; 4049 let Inst{4} = Zt{3}; 4050 let Inst{3} = n; 4051 let Inst{2-0} = Zt{2-0}; 4052 4053 let mayLoad = 1; 4054} 4055 4056class sme2_ld_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, 4057 RegisterOperand multi_vector_ty, 4058 RegisterOperand gpr_ty, 4059 string mnemonic> 4060 : I<(outs multi_vector_ty:$Zt), 4061 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4062 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 4063 "", []>, Sched<[]> { 4064 bits<5> Rm; 4065 bits<3> PNg; 4066 bits<5> Rn; 4067 bits<3> Zt; 4068 let Inst{31-21} = 0b10100001000; 4069 let Inst{20-16} = Rm; 4070 let Inst{15} = 0b1; 4071 let Inst{14-13} = msz; 4072 let Inst{12-10} = PNg; 4073 let Inst{9-5} = Rn; 4074 let Inst{4} = Zt{2}; 4075 let Inst{3} = n; 4076 let Inst{2} = 0b0; 4077 let Inst{1-0} = Zt{1-0}; 4078 4079 let mayLoad = 1; 4080} 4081 4082class sme2_ld_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, 4083 RegisterOperand multi_vector_ty, 4084 Operand index_ty, 4085 string mnemonic> 4086 : I<(outs multi_vector_ty:$Zt), 4087 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), 4088 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]", 4089 "", []>, Sched<[]> { 4090 bits<4> imm4; 4091 bits<3> PNg; 4092 bits<5> Rn; 4093 let Inst{31-20} = 0b101000010100; 4094 let Inst{19-16} = imm4; 4095 let Inst{15} = op{1}; 4096 let Inst{14-13} = msz; 4097 let Inst{12-10} = PNg; 4098 let Inst{9-5} = Rn; 4099 let Inst{3} = n; 4100 let Inst{2} = op{0}; 4101 4102 let mayLoad = 1; 4103} 4104 4105multiclass sme2_ld_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, 4106 RegisterOperand multi_vector_ty, 4107 Operand index_ty, 4108 string mnemonic>{ 4109 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, 4110 multi_vector_ty, 4111 index_ty, mnemonic> { 4112 bits<4> Zt; 4113 let Inst{4} = Zt{3}; 4114 let Inst{2-0} = Zt{2-0}; 4115 } 4116 4117 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", 4118 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 4119} 4120 4121multiclass sme2_ld_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, 4122 RegisterOperand multi_vector_ty, 4123 Operand index_ty, 4124 string mnemonic> { 4125 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, 0b10, 4126 multi_vector_ty, 4127 index_ty, mnemonic> { 4128 bits<3> Zt; 4129 let Inst{4} = Zt{2}; 4130 let Inst{1-0} = Zt{1-0}; 4131 } 4132 4133 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", 4134 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 4135} 4136 4137//===----------------------------------------------------------------------===// 4138// SME2 Non-Contiguous Store 4139class sme2_st_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, 4140 RegisterOperand multi_vector_ty, 4141 RegisterOperand gpr_ty, 4142 string mnemonic> 4143 : I<(outs ), 4144 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4145 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 4146 "", []>, Sched<[]> { 4147 bits<5> Rm; 4148 bits<3> PNg; 4149 bits<5> Rn; 4150 bits<4> Zt; 4151 let Inst{31-21} = 0b10100001001; 4152 let Inst{20-16} = Rm; 4153 let Inst{15} = 0b0; 4154 let Inst{14-13} = msz; 4155 let Inst{12-10} = PNg; 4156 let Inst{9-5} = Rn; 4157 let Inst{4} = Zt{3}; 4158 let Inst{3} = n; 4159 let Inst{2-0} = Zt{2-0}; 4160 4161 let mayStore = 1; 4162} 4163 4164class sme2_st_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, 4165 RegisterOperand multi_vector_ty, 4166 RegisterOperand gpr_ty, 4167 string mnemonic> 4168 : I<(outs ), 4169 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4170 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 4171 "", []>, Sched<[]> { 4172 bits<5> Rm; 4173 bits<3> PNg; 4174 bits<5> Rn; 4175 bits<3> Zt; 4176 let Inst{31-21} = 0b10100001001; 4177 let Inst{20-16} = Rm; 4178 let Inst{15} = 0b1; 4179 let Inst{14-13} = msz; 4180 let Inst{12-10} = PNg; 4181 let Inst{9-5} = Rn; 4182 let Inst{4} = Zt{2}; 4183 let Inst{3} = n; 4184 let Inst{2} = 0b0; 4185 let Inst{1-0} = Zt{1-0}; 4186 4187 let mayStore = 1; 4188} 4189 4190class sme2_st_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, 4191 RegisterOperand multi_vector_ty, 4192 Operand index_ty, 4193 string mnemonic> 4194 : I<(outs ), 4195 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), 4196 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]", 4197 "", []>, Sched<[]> { 4198 bits<4> imm4; 4199 bits<3> PNg; 4200 bits<5> Rn; 4201 let Inst{31-20} = 0b101000010110; 4202 let Inst{19-16} = imm4; 4203 let Inst{15} = op{1}; 4204 let Inst{14-13} = msz; 4205 let Inst{12-10} = PNg; 4206 let Inst{9-5} = Rn; 4207 let Inst{3} = n; 4208 let Inst{2} = op{0}; 4209 4210 let mayStore = 1; 4211} 4212 4213 4214multiclass sme2_st_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, 4215 RegisterOperand multi_vector_ty, 4216 Operand index_ty, 4217 string mnemonic> { 4218 def NAME: sme2_st_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, 4219 multi_vector_ty, 4220 index_ty, mnemonic> { 4221 bits<4> Zt; 4222 let Inst{4} = Zt{3}; 4223 let Inst{2-0} = Zt{2-0}; 4224 } 4225 4226 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", 4227 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 4228} 4229 4230multiclass sme2_st_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, 4231 RegisterOperand multi_vector_ty, 4232 Operand index_ty, 4233 string mnemonic> { 4234 def NAME : sme2_st_vector_vg24_multi_scalar_immediate<msz, n, 0b10, 4235 multi_vector_ty, 4236 index_ty, mnemonic> { 4237 bits<3> Zt; 4238 let Inst{4} = Zt{2}; 4239 let Inst{1-0} = Zt{1-0}; 4240 } 4241 4242 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", 4243 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 4244} 4245 4246//===----------------------------------------------------------------------===// 4247// SME2.1 4248//===----------------------------------------------------------------------===// 4249// SME zeroing move array to vector 4250class sme2p1_movaz_tile_to_vec_base<bits<2> sz, bit q, bit v, ZPRRegOp vector_ty, 4251 RegisterOperand tile_ty, Operand index_ty, 4252 string mnemonic> 4253 : I<(outs vector_ty:$Zd, tile_ty:$ZAn), 4254 (ins tile_ty:$_ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 4255 mnemonic, "\t$Zd, $ZAn[$Rs, $imm]", 4256 "", []>, Sched<[]> { 4257 bits<2> Rs; 4258 bits<5> Zd; 4259 let Inst{31-24} = 0b11000000; 4260 let Inst{23-22} = sz; 4261 let Inst{21-17} = 0b00001; 4262 let Inst{16} = q; 4263 let Inst{15} = v; 4264 let Inst{14-13} = Rs; 4265 let Inst{12-9} = 0b0001; 4266 let Inst{4-0} = Zd; 4267 let Constraints = "$ZAn = $_ZAn"; 4268} 4269 4270multiclass sme2p1_movaz_tile_to_vec_base<bit v, string mnemonic> { 4271 def _B : sme2p1_movaz_tile_to_vec_base<0b00, 0b0, v, ZPR8, 4272 !if(v, TileVectorOpV8, TileVectorOpH8), 4273 sme_elm_idx0_15, mnemonic> { 4274 bits<4> imm; 4275 let Inst{8-5} = imm; 4276 } 4277 4278 def _H : sme2p1_movaz_tile_to_vec_base<0b01, 0b0, v, ZPR16, 4279 !if(v, TileVectorOpV16, TileVectorOpH16), 4280 sme_elm_idx0_7, mnemonic> { 4281 bits<1> ZAn; 4282 bits<3> imm; 4283 let Inst{8} = ZAn; 4284 let Inst{7-5} = imm; 4285 } 4286 4287 def _S : sme2p1_movaz_tile_to_vec_base<0b10, 0b0, v, ZPR32, 4288 !if(v, TileVectorOpV32, TileVectorOpH32), 4289 sme_elm_idx0_3, mnemonic> { 4290 bits<2> ZAn; 4291 bits<2> imm; 4292 let Inst{8-7} = ZAn; 4293 let Inst{6-5} = imm; 4294 } 4295 4296 def _D : sme2p1_movaz_tile_to_vec_base<0b11, 0b0, v, ZPR64, 4297 !if(v, TileVectorOpV64, TileVectorOpH64), 4298 sme_elm_idx0_1, mnemonic> { 4299 bits<3> ZAn; 4300 bits<1> imm; 4301 let Inst{8-6} = ZAn; 4302 let Inst{5} = imm; 4303 } 4304 4305 def _Q : sme2p1_movaz_tile_to_vec_base<0b11, 0b1, v, ZPR128, 4306 !if(v, TileVectorOpV128, TileVectorOpH128), 4307 sme_elm_idx0_0, mnemonic> { 4308 bits<4> ZAn; 4309 let Inst{8-5} = ZAn; 4310 } 4311} 4312 4313multiclass sme2p1_movaz_tile_to_vec<string mnemonic>{ 4314 defm _H : sme2p1_movaz_tile_to_vec_base<0b0, mnemonic>; 4315 defm _V : sme2p1_movaz_tile_to_vec_base<0b1, mnemonic>; 4316} 4317 4318//===----------------------------------------------------------------------===// 4319// SME2.1 multiple vectors zero array 4320 4321class sme2p1_zero_matrix<bits<6> opc, Operand index_ty, string mnemonic, 4322 string vg_acronym=""> 4323 : I<(outs MatrixOp64:$ZAd), 4324 (ins MatrixOp64:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm), 4325 mnemonic, "\t$ZAd[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", 4326 "", []>, Sched<[]> { 4327 bits <2> Rv; 4328 let Inst{31-18} = 0b11000000000011; 4329 let Inst{17-15} = opc{5-3}; 4330 let Inst{14-13} = Rv; 4331 let Inst{12-3} = 0b0000000000; 4332 let Inst{2-0} = opc{2-0}; 4333 let Constraints = "$ZAd = $_ZAd"; 4334} 4335 4336multiclass sme2p1_zero_matrix<string mnemonic> { 4337 def _VG2_Z : sme2p1_zero_matrix<{0b000,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx2"> { 4338 bits<3> imm; 4339 let Inst{2-0} = imm; 4340 } 4341 def _2Z : sme2p1_zero_matrix<{0b001,?,?,?}, uimm3s2range, mnemonic> { 4342 bits<3> imm; 4343 let Inst{2-0} = imm; 4344 } 4345 def _VG2_2Z : sme2p1_zero_matrix<{0b0100,?,?}, uimm2s2range, mnemonic, "vgx2"> { 4346 bits<2> imm; 4347 let Inst{1-0} = imm; 4348 } 4349 def _VG4_2Z : sme2p1_zero_matrix<{0b0110,?,?}, uimm2s2range, mnemonic, "vgx4"> { 4350 bits<2> imm; 4351 let Inst{1-0} = imm; 4352 } 4353 def _VG4_Z : sme2p1_zero_matrix<{0b100,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx4"> { 4354 bits<3> imm; 4355 let Inst{2-0} = imm; 4356 } 4357 def _4Z : sme2p1_zero_matrix<{0b1010,?,?}, uimm2s4range, mnemonic> { 4358 bits<2> imm; 4359 let Inst{1-0} = imm; 4360 } 4361 def _VG2_4Z :sme2p1_zero_matrix<{0b11000,?}, uimm1s4range, mnemonic, "vgx2"> { 4362 bits<1> imm; 4363 let Inst{0} = imm; 4364 } 4365 def _VG4_4Z :sme2p1_zero_matrix<{0b11100,?}, uimm1s4range, mnemonic, "vgx4"> { 4366 bits<1> imm; 4367 let Inst{0} = imm; 4368 } 4369} 4370 4371//===----------------------------------------------------------------------===// 4372// SME2.1 lookup table expand two non-contiguous registers 4373 4374class sme2p1_luti_vector_vg2_index<bits<4> op, bits<2> sz, RegisterOperand vector_ty, 4375 AsmVectorIndexOpnd index_ty, 4376 string mnemonic> 4377 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 4378 mnemonic, "\t$Zd, $ZTt, $Zn$i", 4379 "", []>, Sched<[]> { 4380 bits<5> Zn; 4381 bits<4> Zd; 4382 let Inst{31-19} = 0b1100000010011; 4383 let Inst{18-15} = op; 4384 let Inst{14} = 0b1; 4385 let Inst{13-12} = sz; 4386 let Inst{11-10} = 0b00; 4387 let Inst{9-5} = Zn; 4388 let Inst{4} = Zd{3}; 4389 let Inst{3} = 0b0; 4390 let Inst{2-0} = Zd{2-0}; 4391} 4392 4393class sme2p1_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 4394 AsmVectorIndexOpnd index_ty, 4395 string mnemonic> 4396 : sme2p1_luti_vector_vg2_index<{1,?,?,?}, sz, vector_ty, index_ty, mnemonic> { 4397 bits<3> i; 4398 let Inst{17-15} = i; 4399} 4400 4401multiclass sme2p1_luti2_vector_vg2_index<string mnemonic> { 4402 def _B : sme2p1_luti2_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexH, 4403 mnemonic>; 4404 def _H : sme2p1_luti2_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexH, 4405 mnemonic>; 4406} 4407 4408class sme2p1_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 4409 AsmVectorIndexOpnd index_ty, 4410 string mnemonic> 4411 : sme2p1_luti_vector_vg2_index<{0b01,?,?}, sz, vector_ty, index_ty, mnemonic> { 4412 bits<2> i; 4413 let Inst{16-15} = i; 4414} 4415multiclass sme2p1_luti4_vector_vg2_index<string mnemonic> { 4416 def _B : sme2p1_luti4_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexS, 4417 mnemonic>; 4418 def _H : sme2p1_luti4_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexS, 4419 mnemonic>; 4420} 4421 4422// SME2.1 lookup table expand four non-contiguous registers 4423class sme2p1_luti_vector_vg4_index<bits<3> op, bits<2> sz, RegisterOperand vector_ty, 4424 AsmVectorIndexOpnd index_ty, 4425 string mnemonic> 4426 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 4427 mnemonic, "\t$Zd, $ZTt, $Zn$i", 4428 "", []>, Sched<[]> { 4429 bits<5> Zn; 4430 bits<3> Zd; 4431 let Inst{31-19} = 0b1100000010011; 4432 let Inst{18-16} = op; 4433 let Inst{15-14} = 0b10; 4434 let Inst{13-12} = sz; 4435 let Inst{11-10} = 0b00; 4436 let Inst{9-5} = Zn; 4437 let Inst{4} = Zd{2}; 4438 let Inst{3-2} = 0b00; 4439 let Inst{1-0} = Zd{1-0}; 4440} 4441 4442class sme2p1_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 4443 AsmVectorIndexOpnd index_ty, 4444 string mnemonic> 4445 : sme2p1_luti_vector_vg4_index<{1,?,?}, sz, vector_ty, index_ty, mnemonic> { 4446 bits<2> i; 4447 let Inst{17-16} = i; 4448} 4449 4450multiclass sme2p1_luti2_vector_vg4_index<string mnemonic> { 4451 def _B : sme2p1_luti2_vector_vg4_index<0b00, ZZZZ_b_strided, VectorIndexS, 4452 mnemonic>; 4453 def _H : sme2p1_luti2_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexS, 4454 mnemonic>; 4455} 4456 4457class sme2p1_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 4458 AsmVectorIndexOpnd index_ty, 4459 string mnemonic> 4460 : sme2p1_luti_vector_vg4_index<{0b01,?}, sz, vector_ty, index_ty, mnemonic> { 4461 bit i; 4462 let Inst{16} = i; 4463} 4464 4465multiclass sme2p1_luti4_vector_vg4_index<string mnemonic> { 4466 def _H: sme2p1_luti4_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexD, mnemonic>; 4467} 4468