1//=-- SMEInstrFormats.td - AArch64 SME Instruction classes -*- tablegen -*--=// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// AArch64 Scalable Matrix Extension (SME) Instruction Class Definitions. 10// 11//===----------------------------------------------------------------------===// 12 13def imm_to_tile8 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAB0>", []>; 14def imm_to_tile16 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAH0>", []>; 15def imm_to_tile32 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAS0>", []>; 16def imm_to_tile64 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAD0>", []>; 17def imm_to_tile128 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAQ0>", []>; 18 19def tileslice8 : ComplexPattern<i32 , 2, "SelectSMETileSlice<15, 1>", []>; 20def tileslice16 : ComplexPattern<i32 , 2, "SelectSMETileSlice<7, 1>", []>; 21def tileslice32 : ComplexPattern<i32 , 2, "SelectSMETileSlice<3, 1>", []>; 22def tileslice64 : ComplexPattern<i32 , 2, "SelectSMETileSlice<1, 1>", []>; 23def tileslice128 : ComplexPattern<i32 , 2, "SelectSMETileSlice<0, 1>", []>; // nop 24 25def tileslicerange3s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<14, 2>", []>; 26def tileslicerange2s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<6, 2>", []>; 27def tileslicerange1s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<2, 2>", []>; 28def tileslicerange0s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<0, 2>", []>; 29 30def tileslicerange2s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<12, 4>", []>; 31def tileslicerange1s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<4, 4>", []>; 32def tileslicerange0s4 : ComplexPattern<i32, 2, "SelectSMETileSlice<0, 4>", []>; 33 34def am_sme_indexed_b4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0,15>", [], [SDNPWantRoot]>; 35 36//===----------------------------------------------------------------------===// 37// SME Pseudo Classes 38//===----------------------------------------------------------------------===// 39 40def getSMEPseudoMap : InstrMapping { 41 let FilterClass = "SMEPseudo2Instr"; 42 let RowFields = ["PseudoName"]; 43 let ColFields = ["IsInstr"]; 44 let KeyCol = ["0"]; 45 let ValueCols = [["1"]]; 46} 47 48class SMEPseudo2Instr<string name, bit instr> { 49 string PseudoName = name; 50 bit IsInstr = instr; 51} 52 53class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 54 : Pseudo<(outs), (ins i32imm:$tile, PPR3bAny:$pn, PPR3bAny:$pm, 55 zpr_ty:$zn, zpr_ty:$zm), []>, 56 Sched<[]> { 57 // Translated to the actual instructions in AArch64ISelLowering.cpp 58 let SMEMatrixType = za_flag; 59 let usesCustomInserter = 1; 60} 61 62class sme2_za_array_2op_multi_single_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 63 ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 64 : SMEPseudo2Instr<name, 0>, 65 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), []> { 66 let SMEMatrixType = za_flag; 67 let usesCustomInserter = 1; 68} 69 70class sme2_za_array_2op_multi_multi_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 71 SMEMatrixTypeEnum za_flag> 72 : SMEPseudo2Instr<name, 0>, 73 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), []> { 74 let SMEMatrixType = za_flag; 75 let usesCustomInserter = 1; 76} 77 78class sme2_za_array_2op_multi_index_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, 79 ZPRRegOp zpr_ty, Operand imm_ty, SMEMatrixTypeEnum za_flag> 80 : SMEPseudo2Instr<name, 0>, 81 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, imm_ty:$i), []> { 82 let SMEMatrixType = za_flag; 83 let usesCustomInserter = 1; 84} 85 86class sme2_move_to_za_pseudo<string name, Operand imm_ty, RegisterOperand multi_vector_ty, SMEMatrixTypeEnum za_flag> 87 : SMEPseudo2Instr<name, 0>, 88 Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rs, imm_ty:$imm, multi_vector_ty:$Zn), []> { 89 let SMEMatrixType = za_flag; 90 let usesCustomInserter = 1; 91} 92 93class sme2_move_to_tile_pseudo<string name, Operand tile_imm, Operand imm_ty, RegisterOperand multi_vector_ty, SMEMatrixTypeEnum za_flag> 94 : SMEPseudo2Instr<name, 0>, 95 Pseudo<(outs), (ins tile_imm:$tile, MatrixIndexGPR32Op12_15:$Rs, imm_ty:$imm, multi_vector_ty:$Zn), []> { 96 let SMEMatrixType = za_flag; 97 let usesCustomInserter = 1; 98} 99 100//===----------------------------------------------------------------------===// 101// SME pattern match helpers. 102//===----------------------------------------------------------------------===// 103 104class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 105 ValueType vt, ComplexPattern tileslice> 106 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm), 107 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm)>; 108 109 110class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 111 ValueType vt, ComplexPattern tileslice> 112 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm), 113 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), 114 zpr_ty:$Zm)>; 115class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, 116 ValueType vt, ComplexPattern tileslice> 117 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 118 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm), 119 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 120 (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 121 zpr_ty:$Zm)>; 122 123class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> 124 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2), 125 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 126 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), 127 (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>; 128 129class SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> 130 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 131 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm1, vt:$Zm2, vt:$Zm3, vt:$Zm4), 132 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 133 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 134 (REG_SEQUENCE ZPR4Mul4, vt:$Zm1, zsub0, vt:$Zm2, zsub1, vt:$Zm3, zsub2, vt:$Zm4, zsub3))>; 135 136class SME2_ZA_TwoOp_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 137 Operand imm_ty, ComplexPattern tileslice> 138 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm, (i32 imm_ty:$i)), 139 (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm, (i32 imm_ty:$i))>; 140 141 142class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 143 Operand imm_ty, ComplexPattern tileslice> 144 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)), 145 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 146 (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>; 147 148class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, 149 Operand imm_ty, ComplexPattern tileslice> 150 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), 151 vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)), 152 (!cast<Instruction>(name # _PSEUDO) $base, $offset, 153 (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), 154 zpr_ty:$Zm, imm_ty:$i)>; 155 156class SME2_Sat_Shift_VG2_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty> 157 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, (i32 imm_ty:$i))), 158 (!cast<Instruction>(name) (REG_SEQUENCE ZPR2Mul2, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1), imm_ty:$i)>; 159 160class SME2_Sat_Shift_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt, Operand imm_ty> 161 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4, (i32 imm_ty:$i))), 162 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3), 163 imm_ty:$i)>; 164 165class SME2_Cvt_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt> 166 : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4)), 167 (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3))>; 168 169class SME2_ZA_VG1x2_Multi_Pat<string name, SDPatternOperator intrinsic, ValueType vt, Operand index_ty, ComplexPattern tileslice> 170 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2), 171 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1))>; 172 173class SME2_ZA_VG1x4_Multi_Pat<string name, SDPatternOperator intrinsic, ValueType vt, Operand index_ty, ComplexPattern tileslice> 174 : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4), 175 (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3))>; 176 177class SME2_Tile_VG2_Multi_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt, Operand index_ty, ComplexPattern tileslice> 178 : Pat<(intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2), 179 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1))>; 180 181class SME2_Tile_VG4_Multi_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt, Operand index_ty, ComplexPattern tileslice> 182 : Pat<(intrinsic tile_imm:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4), 183 (!cast<Instruction>(name # _PSEUDO) $tile, $base, $offset, (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3))>; 184 185//===----------------------------------------------------------------------===// 186// SME pattern match helpers. 187//===----------------------------------------------------------------------===// 188 189class SME_ZA_Tile_TwoPred_TwoVec_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType pg_ty, ValueType vt> 190 : Pat<(intrinsic imm_ty:$tile, (pg_ty PPR3bAny:$Pn), (pg_ty PPR3bAny:$Pm), vt:$Zn, vt:$Zm), 191 (!cast<Instruction>(name # _PSEUDO) $tile, $Pn, $Pm, $Zn, $Zm)>; 192 193//===----------------------------------------------------------------------===// 194// SME Outer Products 195//===----------------------------------------------------------------------===// 196 197class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_ty, 198 ZPRRegOp zpr_ty, string mnemonic> 199 : I<(outs za_ty:$ZAda), 200 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 201 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 202 "", []>, 203 Sched<[]> { 204 bits<5> Zm; 205 bits<3> Pm; 206 bits<3> Pn; 207 bits<5> Zn; 208 let Inst{31-25} = 0b1000000; 209 let Inst{24} = op; 210 let Inst{23} = 0b1; 211 let Inst{22-21} = sz; 212 let Inst{20-16} = Zm; 213 let Inst{15-13} = Pm; 214 let Inst{12-10} = Pn; 215 let Inst{9-5} = Zn; 216 let Inst{4} = S; 217 let Inst{3} = op; 218 219 let Constraints = "$ZAda = $_ZAda"; 220} 221 222multiclass sme_outer_product_fp32<bit S, string mnemonic, SDPatternOperator op> { 223 def NAME : sme_fp_outer_product_inst<S, 0b00, 0b0, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { 224 bits<2> ZAda; 225 let Inst{1-0} = ZAda; 226 let Inst{2} = 0b0; 227 } 228 229 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 230 231 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv4i1, nxv4f32>; 232} 233 234multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> { 235 def NAME : sme_fp_outer_product_inst<S, 0b10, 0b0, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { 236 bits<3> ZAda; 237 let Inst{2-0} = ZAda; 238 } 239 240 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 241 242 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv2i1, nxv2f64>; 243} 244 245multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s>{ 246 def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b1, TileOp16, ZPR16, mnemonic> { 247 bits<1> ZAda; 248 let Inst{2-1} = 0b00; 249 let Inst{0} = ZAda; 250 } 251} 252 253class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2, 254 MatrixTileOperand za_ty, ZPRRegOp zpr_ty, 255 string mnemonic> 256 : I<(outs za_ty:$ZAda), 257 (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 258 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 259 "", []>, 260 Sched<[]> { 261 bits<5> Zm; 262 bits<3> Pm; 263 bits<3> Pn; 264 bits<5> Zn; 265 let Inst{31-25} = 0b1010000; 266 let Inst{24} = opc{2}; // u0 267 let Inst{23} = 0b1; 268 let Inst{22} = sz; 269 let Inst{21} = opc{1}; // u1 270 let Inst{20-16} = Zm; 271 let Inst{15-13} = Pm; 272 let Inst{12-10} = Pn; 273 let Inst{9-5} = Zn; 274 let Inst{4} = opc{0}; //S; 275 let Inst{3} = sme2; 276 277 let Constraints = "$ZAda = $_ZAda"; 278} 279 280multiclass sme_int_outer_product_i32<bits<3> opc, string mnemonic, 281 SDPatternOperator op> { 282 def NAME : sme_int_outer_product_inst<opc, 0b0, 0b0, TileOp32, 283 ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> { 284 bits<2> ZAda; 285 let Inst{1-0} = ZAda; 286 let Inst{2} = 0b0; 287 } 288 289 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 290 291 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv16i1, nxv16i8>; 292} 293 294multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic, 295 SDPatternOperator op> { 296 def NAME : sme_int_outer_product_inst<opc, 0b1, 0b0, TileOp64, 297 ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> { 298 bits<3> ZAda; 299 let Inst{2-0} = ZAda; 300 } 301 302 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 303 304 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv8i1, nxv8i16>; 305} 306 307class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic> 308 : I<(outs TileOp32:$ZAda), 309 (ins TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), 310 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", 311 "", []>, 312 Sched<[]> { 313 bits<5> Zm; 314 bits<3> Pm; 315 bits<3> Pn; 316 bits<5> Zn; 317 bits<2> ZAda; 318 let Inst{31-25} = 0b1000000; 319 let Inst{24} = !if(opc{2}, 0, 1); 320 let Inst{23-22} = 0b10; 321 let Inst{21} = opc{1}; 322 let Inst{20-16} = Zm; 323 let Inst{15-13} = Pm; 324 let Inst{12-10} = Pn; 325 let Inst{9-5} = Zn; 326 let Inst{4} = opc{0}; 327 let Inst{3} = opc{2}; 328 let Inst{2} = 0b0; 329 let Inst{1-0} = ZAda; 330 331 let Constraints = "$ZAda = $_ZAda"; 332} 333 334multiclass sme_bf16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { 335 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; 336 337 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 338 339 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv8i1, nxv8bf16>; 340} 341 342multiclass sme_f16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { 343 def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; 344 345 def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 346 347 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_3, nxv8i1, nxv8f16>; 348} 349 350//===----------------------------------------------------------------------===// 351// SME Add Vector to Tile 352//===----------------------------------------------------------------------===// 353 354class sme_add_vector_to_tile_inst<bit op, bit V, MatrixTileOperand tile_ty, 355 ZPRRegOp zpr_ty, string mnemonic> 356 : I<(outs tile_ty:$ZAda), 357 (ins tile_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), 358 mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn", 359 "", []>, Sched<[]> { 360 bits<3> Pm; 361 bits<3> Pn; 362 bits<5> Zn; 363 let Inst{31-23} = 0b110000001; 364 let Inst{22} = op; 365 let Inst{21-17} = 0b01000; 366 let Inst{16} = V; 367 let Inst{15-13} = Pm; 368 let Inst{12-10} = Pn; 369 let Inst{9-5} = Zn; 370 let Inst{4-3} = 0b00; 371 372 let Constraints = "$ZAda = $_ZAda"; 373} 374 375class sme_add_vector_to_tile_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> 376 : Pseudo<(outs), 377 (ins i32imm:$tile, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), []>, 378 Sched<[]> { 379 // Translated to the actual instructions in AArch64ISelLowering.cpp 380 let SMEMatrixType = za_flag; 381 let usesCustomInserter = 1; 382} 383 384multiclass sme_add_vector_to_tile_u32<bit V, string mnemonic, SDPatternOperator op> { 385 def NAME : sme_add_vector_to_tile_inst<0b0, V, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { 386 bits<2> ZAda; 387 let Inst{2} = 0b0; 388 let Inst{1-0} = ZAda; 389 } 390 391 def _PSEUDO_S : sme_add_vector_to_tile_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 392 393 def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm), 394 (nxv4i32 ZPR32:$zn)), 395 (!cast<Instruction>(NAME # _PSEUDO_S) timm32_0_3:$tile, $pn, $pm, $zn)>; 396} 397 398multiclass sme_add_vector_to_tile_u64<bit V, string mnemonic, SDPatternOperator op> { 399 def NAME : sme_add_vector_to_tile_inst<0b1, V, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { 400 bits<3> ZAda; 401 let Inst{2-0} = ZAda; 402 } 403 404 def _PSEUDO_D : sme_add_vector_to_tile_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; 405 406 let Predicates = [HasSMEI16I64] in { 407 def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm), 408 (nxv2i64 ZPR64:$zn)), 409 (!cast<Instruction>(NAME # _PSEUDO_D) timm32_0_7:$tile, $pn, $pm, $zn)>; 410 } 411} 412 413//===----------------------------------------------------------------------===// 414// SME Contiguous Loads 415//===----------------------------------------------------------------------===// 416 417class sme_mem_ld_ss_base<bit Q, bit V, bits<2> msz, dag outs, dag ins, 418 string mnemonic, string argstr> 419 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 420 bits<5> Rm; 421 bits<2> Rv; 422 bits<3> Pg; 423 bits<5> Rn; 424 let Inst{31-25} = 0b1110000; 425 let Inst{24} = Q; 426 let Inst{23-22} = msz; 427 let Inst{21} = 0b0; 428 let Inst{20-16} = Rm; 429 let Inst{15} = V; 430 let Inst{14-13} = Rv; 431 let Inst{12-10} = Pg; 432 let Inst{9-5} = Rn; 433 let Inst{4} = 0b0; 434 435 let mayLoad = 1; 436} 437 438class sme_mem_ld_ss_inst<bit Q, bits<2> msz, string mnemonic, 439 MatrixTileVectorOperand tile_ty, bit is_col, 440 Operand imm_ty, RegisterOperand gpr_ty> 441 : sme_mem_ld_ss_base< 442 Q, is_col, msz, (outs tile_ty:$ZAt), 443 (ins MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, 444 gpr_ty:$Rm), 445 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg/z, [$Rn, $Rm]">; 446 447multiclass sme_mem_ss_aliases_base<string mnemonic, Instruction inst, 448 MatrixTileVectorOperand tile_ty, 449 Operand imm_ty, RegisterOperand gpr_ty, 450 string pg_suffix=""> { 451 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn, $Rm]", 452 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, gpr_ty:$Rm), 0>; 453 // Default XZR offset aliases 454 def : InstAlias<mnemonic # "\t\\{$ZAt[$Rv, $imm]\\}, $Pg" # pg_suffix # ", [$Rn]", 455 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; 456 def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn]", 457 (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; 458} 459 460multiclass sme_mem_ss_aliases<string mnemonic, string inst, bit is_col, 461 string pg_suffix=""> { 462 defm : sme_mem_ss_aliases_base<mnemonic # "b", !cast<Instruction>(inst # _B), 463 !if(is_col, TileVectorOpV8, TileVectorOpH8), 464 sme_elm_idx0_15, GPR64shifted8, pg_suffix>; 465 defm : sme_mem_ss_aliases_base<mnemonic # "h", !cast<Instruction>(inst # _H), 466 !if(is_col, TileVectorOpV16, TileVectorOpH16), 467 sme_elm_idx0_7, GPR64shifted16, pg_suffix>; 468 defm : sme_mem_ss_aliases_base<mnemonic # "w", !cast<Instruction>(inst # _S), 469 !if(is_col, TileVectorOpV32, TileVectorOpH32), 470 sme_elm_idx0_3, GPR64shifted32, pg_suffix>; 471 defm : sme_mem_ss_aliases_base<mnemonic # "d", !cast<Instruction>(inst # _D), 472 !if(is_col, TileVectorOpV64, TileVectorOpH64), 473 sme_elm_idx0_1, GPR64shifted64, pg_suffix>; 474 defm : sme_mem_ss_aliases_base<mnemonic # "q", !cast<Instruction>(inst # _Q), 475 !if(is_col, TileVectorOpV128, TileVectorOpH128), 476 sme_elm_idx0_0, GPR64shifted128, pg_suffix>; 477} 478 479multiclass sme_mem_ld_ss_aliases<string inst, bit is_col> { 480 defm NAME : sme_mem_ss_aliases<"ld1", inst, is_col, "/z">; 481} 482 483multiclass sme_mem_ld_ss_patterns<Instruction Inst, SDPatternOperator Load, 484 Operand tile_ty, Operand offset_ty, 485 ComplexPattern addr, 486 ComplexPattern tileslice> { 487 // base, tileslice 488 def : Pat<(Load PPR3bAny:$pg, GPR64sp:$base, tile_ty:$tile, 489 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 490 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, XZR)>; 491 492 // reg + reg, tileslice 493 let AddedComplexity = 1 in { 494 def : Pat<(Load PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), 495 tile_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 496 offset_ty:$imm))), 497 (Inst tile_ty:$tile, $idx, $imm, $pg, $base, $offset)>; 498 } 499} 500 501class sme_load_pseudo 502 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, 503 i32imm:$imm, PPR3bAny:$pg, GPR64sp:$base, GPR64:$offset), []>, 504 Sched<[]> { 505 // Translated to the actual instructions in AArch64ISelLowering.cpp 506 let usesCustomInserter = 1; 507 let mayLoad = 1; 508} 509 510multiclass sme_mem_ld_v_ss<string mnemonic, bit is_col> { 511 def _B : sme_mem_ld_ss_inst<0b0, 0b00, mnemonic # "b", 512 !if(is_col, TileVectorOpV8, TileVectorOpH8), 513 is_col, sme_elm_idx0_15, GPR64shifted8> { 514 bits<4> imm; 515 let Inst{3-0} = imm; 516 } 517 def _H : sme_mem_ld_ss_inst<0b0, 0b01, mnemonic # "h", 518 !if(is_col, TileVectorOpV16, TileVectorOpH16), 519 is_col, sme_elm_idx0_7, GPR64shifted16> { 520 bits<1> ZAt; 521 bits<3> imm; 522 let Inst{3} = ZAt; 523 let Inst{2-0} = imm; 524 } 525 def _S : sme_mem_ld_ss_inst<0b0, 0b10, mnemonic # "w", 526 !if(is_col, TileVectorOpV32, TileVectorOpH32), 527 is_col, sme_elm_idx0_3, GPR64shifted32> { 528 bits<2> ZAt; 529 bits<2> imm; 530 let Inst{3-2} = ZAt; 531 let Inst{1-0} = imm; 532 } 533 def _D : sme_mem_ld_ss_inst<0b0, 0b11, mnemonic # "d", 534 !if(is_col, TileVectorOpV64, TileVectorOpH64), 535 is_col, sme_elm_idx0_1, GPR64shifted64> { 536 bits<3> ZAt; 537 bits<1> imm; 538 let Inst{3-1} = ZAt; 539 let Inst{0} = imm; 540 } 541 def _Q : sme_mem_ld_ss_inst<0b1, 0b11, mnemonic # "q", 542 !if(is_col, TileVectorOpV128, TileVectorOpH128), 543 is_col, sme_elm_idx0_0, GPR64shifted128> { 544 bits<4> ZAt; 545 let Inst{3-0} = ZAt; 546 } 547 548 defm : sme_mem_ld_ss_aliases<NAME, is_col>; 549 550 // Pseudo instructions for lowering intrinsics, using immediates instead of 551 // tile registers. 552 def _PSEUDO_B : sme_load_pseudo; 553 def _PSEUDO_H : sme_load_pseudo; 554 def _PSEUDO_S : sme_load_pseudo; 555 def _PSEUDO_D : sme_load_pseudo; 556 def _PSEUDO_Q : sme_load_pseudo; 557 558 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_B), 559 !if(is_col, int_aarch64_sme_ld1b_vert, 560 int_aarch64_sme_ld1b_horiz), 561 sme_elm_idx0_0, timm32_0_15, am_sve_regreg_lsl0, 562 tileslice8>; 563 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 564 !if(is_col, int_aarch64_sme_ld1h_vert, 565 int_aarch64_sme_ld1h_horiz), 566 timm32_0_1, timm32_0_7, am_sve_regreg_lsl1, 567 tileslice16>; 568 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 569 !if(is_col, int_aarch64_sme_ld1w_vert, 570 int_aarch64_sme_ld1w_horiz), 571 timm32_0_3, timm32_0_3, am_sve_regreg_lsl2, 572 tileslice32>; 573 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 574 !if(is_col, int_aarch64_sme_ld1d_vert, 575 int_aarch64_sme_ld1d_horiz), 576 timm32_0_7, timm32_0_1, am_sve_regreg_lsl3, 577 tileslice64>; 578 defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 579 !if(is_col, int_aarch64_sme_ld1q_vert, 580 int_aarch64_sme_ld1q_horiz), 581 timm32_0_15, sme_elm_idx0_0, am_sve_regreg_lsl4, 582 tileslice128>; 583} 584 585multiclass sme_mem_ld_ss<string mnemonic> { 586 defm _H : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b0>; 587 defm _V : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b1>; 588} 589 590//===----------------------------------------------------------------------===// 591// SME Contiguous Stores 592//===----------------------------------------------------------------------===// 593 594class sme_mem_st_ss_base<bit Q, bit V, bits<2> msz, dag ins, 595 string mnemonic, string argstr> 596 : I<(outs), ins, mnemonic, argstr, "", []>, Sched<[]> { 597 bits<5> Rm; 598 bits<2> Rv; 599 bits<3> Pg; 600 bits<5> Rn; 601 let Inst{31-25} = 0b1110000; 602 let Inst{24} = Q; 603 let Inst{23-22} = msz; 604 let Inst{21} = 0b1; 605 let Inst{20-16} = Rm; 606 let Inst{15} = V; 607 let Inst{14-13} = Rv; 608 let Inst{12-10} = Pg; 609 let Inst{9-5} = Rn; 610 let Inst{4} = 0b0; 611 612 let mayStore = 1; 613 let hasSideEffects = 1; 614} 615 616class sme_mem_st_ss_inst<bit Q, bits<2> msz, string mnemonic, 617 MatrixTileVectorOperand tile_ty, bit is_col, 618 Operand imm_ty, RegisterOperand gpr_ty> 619 : sme_mem_st_ss_base< 620 Q, is_col, msz, 621 (ins tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, 622 GPR64sp:$Rn, gpr_ty:$Rm), 623 mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg, [$Rn, $Rm]">; 624 625multiclass sme_mem_st_ss_aliases<string inst, bit is_col> { 626 defm NAME : sme_mem_ss_aliases<"st1", inst, is_col>; 627} 628 629multiclass sme_mem_st_ss_patterns<Instruction Inst, SDPatternOperator Store, 630 Operand offset_ty, 631 ComplexPattern imm2tile, 632 ComplexPattern addr, 633 ComplexPattern tileslice> { 634 // base, tileslice 635 def : Pat<(Store PPR3bAny:$pg, GPR64sp:$base, (imm2tile untyped:$tile), 636 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 637 (Inst $tile, $idx, $imm, $pg, $base, XZR)>; 638 639 // reg + reg, tileslice 640 let AddedComplexity = 1 in { 641 def : Pat<(Store PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), 642 (imm2tile untyped:$tile), 643 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), 644 (Inst $tile, $idx, $imm, $pg, $base, $offset)>; 645 } 646} 647 648multiclass sme_mem_st_v_ss<string mnemonic, bit is_col> { 649 def _B : sme_mem_st_ss_inst<0b0, 0b00, mnemonic # "b", 650 !if(is_col, TileVectorOpV8, TileVectorOpH8), 651 is_col, sme_elm_idx0_15, GPR64shifted8> { 652 bits<4> imm; 653 let Inst{3-0} = imm; 654 } 655 def _H : sme_mem_st_ss_inst<0b0, 0b01, mnemonic # "h", 656 !if(is_col, TileVectorOpV16, TileVectorOpH16), 657 is_col, sme_elm_idx0_7, GPR64shifted16> { 658 bits<1> ZAt; 659 bits<3> imm; 660 let Inst{3} = ZAt; 661 let Inst{2-0} = imm; 662 } 663 def _S : sme_mem_st_ss_inst<0b0, 0b10, mnemonic # "w", 664 !if(is_col, TileVectorOpV32, TileVectorOpH32), 665 is_col, sme_elm_idx0_3, GPR64shifted32> { 666 bits<2> ZAt; 667 bits<2> imm; 668 let Inst{3-2} = ZAt; 669 let Inst{1-0} = imm; 670 } 671 def _D : sme_mem_st_ss_inst<0b0, 0b11, mnemonic # "d", 672 !if(is_col, TileVectorOpV64, TileVectorOpH64), 673 is_col, sme_elm_idx0_1, GPR64shifted64> { 674 bits<3> ZAt; 675 bits<1> imm; 676 let Inst{3-1} = ZAt; 677 let Inst{0} = imm; 678 } 679 def _Q : sme_mem_st_ss_inst<0b1, 0b11, mnemonic # "q", 680 !if(is_col, TileVectorOpV128, TileVectorOpH128), 681 is_col, sme_elm_idx0_0, GPR64shifted128> { 682 bits<4> ZAt; 683 let Inst{3-0} = ZAt; 684 } 685 686 defm : sme_mem_st_ss_aliases<NAME, is_col>; 687 688 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _B), 689 !if(is_col, int_aarch64_sme_st1b_vert, 690 int_aarch64_sme_st1b_horiz), 691 timm32_0_15, imm_to_tile8, am_sve_regreg_lsl0, 692 tileslice8>; 693 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _H), 694 !if(is_col, int_aarch64_sme_st1h_vert, 695 int_aarch64_sme_st1h_horiz), 696 timm32_0_7, imm_to_tile16, am_sve_regreg_lsl1, 697 tileslice16>; 698 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _S), 699 !if(is_col, int_aarch64_sme_st1w_vert, 700 int_aarch64_sme_st1w_horiz), 701 timm32_0_3, imm_to_tile32, am_sve_regreg_lsl2, 702 tileslice32>; 703 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _D), 704 !if(is_col, int_aarch64_sme_st1d_vert, 705 int_aarch64_sme_st1d_horiz), 706 timm32_0_1, imm_to_tile64, am_sve_regreg_lsl3, 707 tileslice64>; 708 defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _Q), 709 !if(is_col, int_aarch64_sme_st1q_vert, 710 int_aarch64_sme_st1q_horiz), 711 sme_elm_idx0_0, imm_to_tile128, 712 am_sve_regreg_lsl4, tileslice128>; 713} 714 715multiclass sme_mem_st_ss<string mnemonic> { 716 defm _H : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b0>; 717 defm _V : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b1>; 718} 719 720//===----------------------------------------------------------------------===// 721// SME Save and Restore Array 722//===----------------------------------------------------------------------===// 723 724class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr> 725 : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "", 726 []>, 727 Sched<[]> { 728 bits<2> Rv; 729 bits<5> Rn; 730 bits<4> imm4; 731 let Inst{31-22} = 0b1110000100; 732 let Inst{21} = isStore; 733 let Inst{20-15} = 0b000000; 734 let Inst{14-13} = Rv; 735 let Inst{12-10} = 0b000; 736 let Inst{9-5} = Rn; 737 let Inst{4} = 0b0; 738 let Inst{3-0} = imm4; 739} 740 741let mayStore = 1 in 742class sme_spill_inst<string opcodestr> 743 : sme_spill_fill_base<0b1, (outs), 744 (ins MatrixOp:$ZAt, MatrixIndexGPR32Op12_15:$Rv, 745 sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 746 imm0_15:$offset), 747 opcodestr>; 748let mayLoad = 1 in 749class sme_fill_inst<string opcodestr> 750 : sme_spill_fill_base<0b0, (outs MatrixOp:$ZAt), 751 (ins MatrixIndexGPR32Op12_15:$Rv, 752 sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 753 imm0_15:$offset), 754 opcodestr>; 755multiclass sme_spill<string opcodestr> { 756 def NAME : sme_spill_inst<opcodestr>; 757 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", 758 (!cast<Instruction>(NAME) MatrixOp:$ZAt, 759 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; 760 // base 761 def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base), 762 (!cast<Instruction>(NAME) ZA, $idx, 0, $base, 0)>; 763} 764 765multiclass sme_fill<string opcodestr> { 766 def NAME : sme_fill_inst<opcodestr>; 767 def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", 768 (!cast<Instruction>(NAME) MatrixOp:$ZAt, 769 MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; 770 def NAME # _PSEUDO 771 : Pseudo<(outs), 772 (ins MatrixIndexGPR32Op12_15:$idx, imm0_15:$imm4, 773 GPR64sp:$base), []>, 774 Sched<[]> { 775 // Translated to actual instruction in AArch64ISelLowering.cpp 776 let usesCustomInserter = 1; 777 let mayLoad = 1; 778 } 779 // base 780 def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base), 781 (!cast<Instruction>(NAME # _PSEUDO) $idx, 0, $base)>; 782} 783 784//===----------------------------------------------------------------------===// 785// Move instructions 786//===----------------------------------------------------------------------===// 787 788class sme_vector_to_tile_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, 789 string mnemonic, string argstr> 790 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 791 bits<2> Rv; 792 bits<3> Pg; 793 bits<5> Zn; 794 let Inst{31-24} = 0b11000000; 795 let Inst{23-22} = sz; 796 let Inst{21-17} = 0b00000; 797 let Inst{16} = Q; 798 let Inst{15} = V; 799 let Inst{14-13} = Rv; 800 let Inst{12-10} = Pg; 801 let Inst{9-5} = Zn; 802 let Inst{4} = 0b0; 803} 804 805class sme_vector_to_tile_inst<bit Q, bits<2> sz, MatrixTileVectorOperand tile_ty, 806 bit is_col, Operand imm_ty, ZPRRegOp zpr_ty, 807 string mnemonic> 808 : sme_vector_to_tile_base<Q, is_col, sz, (outs tile_ty:$ZAd), 809 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 810 mnemonic, "\t$ZAd[$Rv, $imm], $Pg/m, $Zn">{ 811 812 let Constraints = "$ZAd = $_ZAd"; 813} 814 815 816multiclass sme_vector_to_tile_aliases<Instruction inst, 817 MatrixTileVectorOperand tile_ty, 818 ZPRRegOp zpr_ty, Operand imm_ty> { 819 def : InstAlias<"mov\t$ZAd[$Rv, $imm], $Pg/m, $Zn", 820 (inst tile_ty:$ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 1>; 821} 822 823multiclass sme_vector_to_tile_patterns<Instruction inst, ValueType zpr_vt, 824 ValueType ppr_vt, Operand imm_ty, 825 Operand offset_ty, 826 SDPatternOperator op, 827 ComplexPattern tileslice> { 828 def : Pat<(op imm_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 829 offset_ty:$imm)), 830 (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)), 831 (inst imm_ty:$tile, $idx, $imm, $pg, $zn)>; 832} 833 834class sme_mova_insert_pseudo<SMEMatrixTypeEnum za_flag> 835 : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, 836 i32imm:$imm, PPR3bAny:$pg, ZPRAny:$zn), []>, 837 Sched<[]> { 838 // Translated to the actual instructions in AArch64ISelLowering.cpp 839 let SMEMatrixType = za_flag; 840 let usesCustomInserter = 1; 841} 842 843multiclass sme_vector_v_to_tile<string mnemonic, bit is_col> { 844 def _B : sme_vector_to_tile_inst<0b0, 0b00, !if(is_col, TileVectorOpV8, 845 TileVectorOpH8), 846 is_col, sme_elm_idx0_15, ZPR8, mnemonic>, 847 SMEPseudo2Instr<NAME # _B, 1> { 848 bits<4> imm; 849 let Inst{3-0} = imm; 850 } 851 def _H : sme_vector_to_tile_inst<0b0, 0b01, !if(is_col, TileVectorOpV16, 852 TileVectorOpH16), 853 is_col, sme_elm_idx0_7, ZPR16, mnemonic>, 854 SMEPseudo2Instr<NAME # _H, 1> { 855 bits<1> ZAd; 856 bits<3> imm; 857 let Inst{3} = ZAd; 858 let Inst{2-0} = imm; 859 } 860 def _S : sme_vector_to_tile_inst<0b0, 0b10, !if(is_col, TileVectorOpV32, 861 TileVectorOpH32), 862 is_col, sme_elm_idx0_3, ZPR32, mnemonic>, 863 SMEPseudo2Instr<NAME # _S, 1> { 864 bits<2> ZAd; 865 bits<2> imm; 866 let Inst{3-2} = ZAd; 867 let Inst{1-0} = imm; 868 } 869 def _D : sme_vector_to_tile_inst<0b0, 0b11, !if(is_col, TileVectorOpV64, 870 TileVectorOpH64), 871 is_col, sme_elm_idx0_1, ZPR64, mnemonic>, 872 SMEPseudo2Instr<NAME # _D, 1> { 873 bits<3> ZAd; 874 bits<1> imm; 875 let Inst{3-1} = ZAd; 876 let Inst{0} = imm; 877 } 878 def _Q : sme_vector_to_tile_inst<0b1, 0b11, !if(is_col, TileVectorOpV128, 879 TileVectorOpH128), 880 is_col, sme_elm_idx0_0, ZPR128, mnemonic>, 881 SMEPseudo2Instr<NAME # _Q, 1> { 882 bits<4> ZAd; 883 bits<1> imm; 884 let Inst{3-0} = ZAd; 885 } 886 887 // Pseudo instructions for lowering intrinsics, using immediates instead of 888 // tile registers. 889 def _PSEUDO_B : sme_mova_insert_pseudo<SMEMatrixTileB>, SMEPseudo2Instr<NAME # _B, 0>; 890 def _PSEUDO_H : sme_mova_insert_pseudo<SMEMatrixTileH>, SMEPseudo2Instr<NAME # _H, 0>; 891 def _PSEUDO_S : sme_mova_insert_pseudo<SMEMatrixTileS>, SMEPseudo2Instr<NAME # _S, 0>; 892 def _PSEUDO_D : sme_mova_insert_pseudo<SMEMatrixTileD>, SMEPseudo2Instr<NAME # _D, 0>; 893 def _PSEUDO_Q : sme_mova_insert_pseudo<SMEMatrixTileQ>, SMEPseudo2Instr<NAME # _Q, 0>; 894 895 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _B), 896 !if(is_col, TileVectorOpV8, 897 TileVectorOpH8), 898 ZPR8, sme_elm_idx0_15>; 899 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _H), 900 !if(is_col, TileVectorOpV16, 901 TileVectorOpH16), 902 ZPR16, sme_elm_idx0_7>; 903 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _S), 904 !if(is_col, TileVectorOpV32, 905 TileVectorOpH32), 906 ZPR32, sme_elm_idx0_3>; 907 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _D), 908 !if(is_col, TileVectorOpV64, 909 TileVectorOpH64), 910 ZPR64, sme_elm_idx0_1>; 911 defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _Q), 912 !if(is_col, TileVectorOpV128, 913 TileVectorOpH128), 914 ZPR128, sme_elm_idx0_0>; 915 916 defvar op = !if(is_col, int_aarch64_sme_write_vert, 917 int_aarch64_sme_write_horiz); 918 919 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_B), 920 nxv16i8, nxv16i1, sme_elm_idx0_0, sme_elm_idx0_15, 921 op, tileslice8>; 922 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 923 nxv8i16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 924 op, tileslice16>; 925 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 926 nxv8f16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 927 op, tileslice16>; 928 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), 929 nxv8bf16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, 930 op, tileslice16>; 931 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 932 nxv4i32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, 933 op, tileslice32>; 934 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), 935 nxv4f32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, 936 op, tileslice32>; 937 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 938 nxv2i64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, 939 op, tileslice64>; 940 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), 941 nxv2f64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, 942 op, tileslice64>; 943 944 defvar opq = !if(is_col, int_aarch64_sme_writeq_vert, 945 int_aarch64_sme_writeq_horiz); 946 947 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 948 nxv16i8, nxv16i1, sme_elm_idx0_15, 949 sme_elm_idx0_0, opq, tileslice128>; 950 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 951 nxv8i16, nxv8i1, sme_elm_idx0_15, 952 sme_elm_idx0_0, opq, tileslice128>; 953 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 954 nxv8f16, nxv8i1, sme_elm_idx0_15, 955 sme_elm_idx0_0, opq, tileslice128>; 956 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 957 nxv8bf16, nxv8i1, sme_elm_idx0_15, 958 sme_elm_idx0_0, opq, tileslice128>; 959 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 960 nxv4i32, nxv4i1, sme_elm_idx0_15, 961 sme_elm_idx0_0, opq, tileslice128>; 962 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 963 nxv4f32, nxv4i1, sme_elm_idx0_15, 964 sme_elm_idx0_0, opq, tileslice128>; 965 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 966 nxv2i64, nxv2i1, sme_elm_idx0_15, 967 sme_elm_idx0_0, opq, tileslice128>; 968 defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), 969 nxv2f64, nxv2i1, sme_elm_idx0_15, 970 sme_elm_idx0_0, opq, tileslice128>; 971} 972 973multiclass sme_vector_to_tile<string mnemonic> { 974 defm _H : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b0>; 975 defm _V : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b1>; 976} 977 978class sme_tile_to_vector_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, 979 string mnemonic, string argstr> 980 : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { 981 bits<2> Rv; 982 bits<3> Pg; 983 bits<5> Zd; 984 let Inst{31-24} = 0b11000000; 985 let Inst{23-22} = sz; 986 let Inst{21-17} = 0b00001; 987 let Inst{16} = Q; 988 let Inst{15} = V; 989 let Inst{14-13} = Rv; 990 let Inst{12-10} = Pg; 991 let Inst{9} = 0b0; 992 let Inst{4-0} = Zd; 993} 994 995class sme_tile_to_vector_inst<bit Q, bits<2> sz, ZPRRegOp zpr_ty, 996 MatrixTileVectorOperand tile_ty, 997 bit is_col, Operand imm_ty, string mnemonic> 998 : sme_tile_to_vector_base<Q, is_col, sz, (outs zpr_ty:$Zd), 999 (ins zpr_ty:$_Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1000 mnemonic, "\t$Zd, $Pg/m, $ZAn[$Rv, $imm]"> { 1001 1002 let Constraints = "$Zd = $_Zd"; 1003} 1004 1005multiclass sme_tile_to_vector_aliases<Instruction inst, ZPRRegOp zpr_ty, 1006 MatrixTileVectorOperand tile_ty, 1007 Operand imm_ty > { 1008 def : InstAlias<"mov\t$Zd, $Pg/m, $ZAn[$Rv, $imm]", 1009 (inst zpr_ty:$Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1>; 1010} 1011 1012multiclass sme_tile_to_vector_patterns<Instruction inst, ValueType zpr_vt, 1013 ValueType ppr_vt, Operand offset_ty, 1014 ComplexPattern imm2tile, 1015 ComplexPattern tileslice, 1016 SDPatternOperator op> { 1017 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), 1018 (imm2tile untyped:$tile), MatrixIndexGPR32Op12_15:$idx)), 1019 (inst $passthru, $pg, $tile, $idx, 0)>; 1020 let AddedComplexity = 1 in { 1021 def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), 1022 (imm2tile untyped:$tile), 1023 (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, 1024 offset_ty:$imm)))), 1025 (inst $passthru, $pg, $tile, $idx, $imm)>; 1026 } 1027} 1028 1029multiclass sme_tile_to_vector_v<string mnemonic, bit is_col> { 1030 def _B : sme_tile_to_vector_inst<0b0, 0b00, ZPR8, !if(is_col, TileVectorOpV8, 1031 TileVectorOpH8), 1032 is_col, sme_elm_idx0_15, mnemonic> { 1033 bits<4> imm; 1034 let Inst{8-5} = imm; 1035 } 1036 def _H : sme_tile_to_vector_inst<0b0, 0b01, ZPR16, !if(is_col, TileVectorOpV16, 1037 TileVectorOpH16), 1038 is_col, sme_elm_idx0_7, mnemonic> { 1039 bits<1> ZAn; 1040 bits<3> imm; 1041 let Inst{8} = ZAn; 1042 let Inst{7-5} = imm; 1043 } 1044 def _S : sme_tile_to_vector_inst<0b0, 0b10, ZPR32, !if(is_col, TileVectorOpV32, 1045 TileVectorOpH32), 1046 is_col, sme_elm_idx0_3, mnemonic> { 1047 bits<2> ZAn; 1048 bits<2> imm; 1049 let Inst{8-7} = ZAn; 1050 let Inst{6-5} = imm; 1051 } 1052 def _D : sme_tile_to_vector_inst<0b0, 0b11, ZPR64, !if(is_col, TileVectorOpV64, 1053 TileVectorOpH64), 1054 is_col, sme_elm_idx0_1, mnemonic> { 1055 bits<3> ZAn; 1056 bits<1> imm; 1057 let Inst{8-6} = ZAn; 1058 let Inst{5} = imm; 1059 } 1060 def _Q : sme_tile_to_vector_inst<0b1, 0b11, ZPR128, !if(is_col, TileVectorOpV128, 1061 TileVectorOpH128), 1062 is_col, sme_elm_idx0_0, mnemonic> { 1063 bits<4> ZAn; 1064 let Inst{8-5} = ZAn; 1065 } 1066 1067 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _B), ZPR8, 1068 !if(is_col, TileVectorOpV8, 1069 TileVectorOpH8), sme_elm_idx0_15>; 1070 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _H), ZPR16, 1071 !if(is_col, TileVectorOpV16, 1072 TileVectorOpH16), sme_elm_idx0_7>; 1073 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _S), ZPR32, 1074 !if(is_col, TileVectorOpV32, 1075 TileVectorOpH32), sme_elm_idx0_3>; 1076 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _D), ZPR64, 1077 !if(is_col, TileVectorOpV64, 1078 TileVectorOpH64), sme_elm_idx0_1>; 1079 defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _Q), ZPR128, 1080 !if(is_col, TileVectorOpV128, 1081 TileVectorOpH128), sme_elm_idx0_0>; 1082 1083 defvar op = !if(is_col, int_aarch64_sme_read_vert, 1084 int_aarch64_sme_read_horiz); 1085 1086 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _B), 1087 nxv16i8, nxv16i1, sme_elm_idx0_15, 1088 imm_to_tile8, tileslice8, op>; 1089 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1090 nxv8i16, nxv8i1, sme_elm_idx0_7, 1091 imm_to_tile16, tileslice16, op>; 1092 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1093 nxv8f16, nxv8i1, sme_elm_idx0_7, 1094 imm_to_tile16, tileslice16, op>; 1095 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), 1096 nxv8bf16, nxv8i1, sme_elm_idx0_7, 1097 imm_to_tile16, tileslice16, op>; 1098 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), 1099 nxv4i32, nxv4i1, sme_elm_idx0_3, 1100 imm_to_tile32, tileslice32, op>; 1101 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), 1102 nxv4f32, nxv4i1, sme_elm_idx0_3, 1103 imm_to_tile32, tileslice32, op>; 1104 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), 1105 nxv2i64, nxv2i1, sme_elm_idx0_1, 1106 imm_to_tile64, tileslice64, op>; 1107 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), 1108 nxv2f64, nxv2i1, sme_elm_idx0_1, 1109 imm_to_tile64, tileslice64, op>; 1110 1111 defvar opq = !if(is_col, int_aarch64_sme_readq_vert, 1112 int_aarch64_sme_readq_horiz); 1113 1114 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1115 nxv16i8, nxv16i1, sme_elm_idx0_0, 1116 imm_to_tile128, tileslice128, opq>; 1117 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1118 nxv8i16, nxv8i1, sme_elm_idx0_0, 1119 imm_to_tile128, tileslice128, opq>; 1120 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1121 nxv8f16, nxv8i1, sme_elm_idx0_0, 1122 imm_to_tile128, tileslice128, opq>; 1123 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1124 nxv8bf16, nxv8i1, sme_elm_idx0_0, 1125 imm_to_tile128, tileslice128, opq>; 1126 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1127 nxv4i32, nxv4i1, sme_elm_idx0_0, 1128 imm_to_tile128, tileslice128, opq>; 1129 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1130 nxv4f32, nxv4i1, sme_elm_idx0_0, 1131 imm_to_tile128, tileslice128, opq>; 1132 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1133 nxv2i64, nxv2i1, sme_elm_idx0_0, 1134 imm_to_tile128, tileslice128, opq>; 1135 defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), 1136 nxv2f64, nxv2i1, sme_elm_idx0_0, 1137 imm_to_tile128, tileslice128, opq>; 1138} 1139 1140multiclass sme_tile_to_vector<string mnemonic> { 1141 defm _H : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b0>; 1142 defm _V : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b1>; 1143} 1144 1145//===----------------------------------------------------------------------===// 1146// SME Zero 1147//===----------------------------------------------------------------------===// 1148 1149// NOTE: This definition isn't really correct because there are outputs, i.e. 1150// the tile registers being zeroed. We fix this up in a custom inserter that 1151// marks the appropriate registers as being implicitly defined. 1152class sme_zero_inst<string mnemonic> 1153 : I<(outs), (ins MatrixTileList:$imm), 1154 mnemonic, "\t$imm", "", []>, Sched<[]> { 1155 bits<8> imm; 1156 let Inst{31-8} = 0b110000000000100000000000; 1157 let Inst{7-0} = imm; 1158} 1159 1160multiclass sme_zero<string mnemonic> { 1161 def NAME : sme_zero_inst<mnemonic>; 1162 1163 def : InstAlias<"zero\t\\{za\\}", (!cast<Instruction>(NAME) 0b11111111), 1>; 1164 def : InstAlias<"zero\t\\{za0.h\\}", (!cast<Instruction>(NAME) 0b01010101), 1>; 1165 def : InstAlias<"zero\t\\{za1.h\\}", (!cast<Instruction>(NAME) 0b10101010), 1>; 1166 def : InstAlias<"zero\t\\{za0.s\\}", (!cast<Instruction>(NAME) 0b00010001), 1>; 1167 def : InstAlias<"zero\t\\{za1.s\\}", (!cast<Instruction>(NAME) 0b00100010), 1>; 1168 def : InstAlias<"zero\t\\{za2.s\\}", (!cast<Instruction>(NAME) 0b01000100), 1>; 1169 def : InstAlias<"zero\t\\{za3.s\\}", (!cast<Instruction>(NAME) 0b10001000), 1>; 1170 def : InstAlias<"zero\t\\{za0.s,za1.s\\}", (!cast<Instruction>(NAME) 0b00110011), 1>; 1171 def : InstAlias<"zero\t\\{za0.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10011001), 1>; 1172 def : InstAlias<"zero\t\\{za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01100110), 1>; 1173 def : InstAlias<"zero\t\\{za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11001100), 1>; 1174 def : InstAlias<"zero\t\\{za0.s,za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01110111), 1>; 1175 def : InstAlias<"zero\t\\{za0.s,za1.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10111011), 1>; 1176 def : InstAlias<"zero\t\\{za0.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11011101), 1>; 1177 def : InstAlias<"zero\t\\{za1.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11101110), 1>; 1178 1179 def NAME # _PSEUDO : Pseudo<(outs), (ins i32imm:$tilelist), []>, 1180 Sched<[]> { 1181 // Translated to the actual instructions in AArch64ISelLowering.cpp 1182 let usesCustomInserter = 1; 1183 } 1184 1185 def : Pat<(int_aarch64_sme_zero timm32_0_255:$imm), 1186 (!cast<Instruction>(NAME # _PSEUDO) timm32_0_255:$imm)>; 1187} 1188 1189//===----------------------------------------------------------------------===// 1190// SVE2 Instructions 1191//===----------------------------------------------------------------------===// 1192 1193class sve2_int_perm_revd<string asm> 1194 : I<(outs ZPR128:$Zd), (ins ZPR128:$_Zd, PPR3bAny:$Pg, ZPR128:$Zn), 1195 asm, "\t$Zd, $Pg/m, $Zn", "", []>, 1196 Sched<[]> { 1197 bits<5> Zd; 1198 bits<3> Pg; 1199 bits<5> Zn; 1200 let Inst{31-24} = 0b00000101; 1201 let Inst{23-22} = 0b00; // size 1202 let Inst{21-13} = 0b101110100; 1203 let Inst{12-10} = Pg; 1204 let Inst{9-5} = Zn; 1205 let Inst{4-0} = Zd; 1206 1207 let Constraints = "$Zd = $_Zd"; 1208 let DestructiveInstType = DestructiveUnary; 1209 let ElementSize = ZPR128.ElementSize; 1210} 1211 1212multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> { 1213 def NAME : sve2_int_perm_revd<asm>; 1214 1215 def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>; 1216 def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>; 1217 def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>; 1218 def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>; 1219} 1220 1221class sve2_clamp<string asm, bits<2> sz, bit U, ZPRRegOp zpr_ty> 1222 : I<(outs zpr_ty:$Zd), (ins zpr_ty:$Zn, zpr_ty:$Zm, zpr_ty:$_Zd), 1223 asm, "\t$Zd, $Zn, $Zm", "", []>, 1224 Sched<[]> { 1225 bits<5> Zm; 1226 bits<5> Zn; 1227 bits<5> Zd; 1228 let Inst{31-24} = 0b01000100; 1229 let Inst{23-22} = sz; 1230 let Inst{21} = 0b0; 1231 let Inst{20-16} = Zm; 1232 let Inst{15-11} = 0b11000; 1233 let Inst{10} = U; 1234 let Inst{9-5} = Zn; 1235 let Inst{4-0} = Zd; 1236 1237 let Constraints = "$Zd = $_Zd"; 1238 let DestructiveInstType = DestructiveOther; 1239 let ElementSize = zpr_ty.ElementSize; 1240} 1241 1242multiclass sve2_clamp<string asm, bit U, SDPatternOperator op> { 1243 def _B : sve2_clamp<asm, 0b00, U, ZPR8>; 1244 def _H : sve2_clamp<asm, 0b01, U, ZPR16>; 1245 def _S : sve2_clamp<asm, 0b10, U, ZPR32>; 1246 def _D : sve2_clamp<asm, 0b11, U, ZPR64>; 1247 1248 def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; 1249 def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; 1250 def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; 1251 def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; 1252} 1253 1254class sve2_int_perm_sel_p<string asm, PPRRegOp ppr_ty, Operand imm_ty> 1255 : I<(outs PPRAny:$Pd), (ins PPRAny:$Pn, ppr_ty:$Pm, 1256 MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1257 asm, "\t$Pd, $Pn, $Pm[$Rv, $imm]", "", []>, 1258 Sched<[]> { 1259 bits<2> Rv; 1260 bits<4> Pn; 1261 bits<4> Pm; 1262 bits<4> Pd; 1263 let Inst{31-24} = 0b00100101; 1264 let Inst{21} = 0b1; 1265 let Inst{17-16} = Rv; 1266 let Inst{15-14} = 0b01; 1267 let Inst{13-10} = Pn; 1268 let Inst{9} = 0b0; 1269 let Inst{8-5} = Pm; 1270 let Inst{4} = 0b0; 1271 let Inst{3-0} = Pd; 1272} 1273 1274multiclass sve2_int_perm_sel_p<string asm, SDPatternOperator op> { 1275 def _B : sve2_int_perm_sel_p<asm, PPR8, sme_elm_idx0_15> { 1276 bits<4> imm; 1277 let Inst{23-22} = imm{3-2}; 1278 let Inst{20-19} = imm{1-0}; 1279 let Inst{18} = 0b1; 1280 } 1281 def _H : sve2_int_perm_sel_p<asm, PPR16, sme_elm_idx0_7> { 1282 bits<3> imm; 1283 let Inst{23-22} = imm{2-1}; 1284 let Inst{20} = imm{0}; 1285 let Inst{19-18} = 0b10; 1286 } 1287 def _S : sve2_int_perm_sel_p<asm, PPR32, sme_elm_idx0_3> { 1288 bits<2> imm; 1289 let Inst{23-22} = imm{1-0}; 1290 let Inst{20-18} = 0b100; 1291 } 1292 def _D : sve2_int_perm_sel_p<asm, PPR64, sme_elm_idx0_1> { 1293 bits<1> imm; 1294 let Inst{23} = imm; 1295 let Inst{22} = 0b1; 1296 let Inst{20-18} = 0b000; 1297 } 1298 1299 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1300 (!cast<Instruction>(NAME # _B) PNRAny:$Pd, 1301 PNRAny:$Pn, PPR8:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm), 0>; 1302 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1303 (!cast<Instruction>(NAME # _H) PNRAny:$Pd, 1304 PNRAny:$Pn, PPR16:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_7:$imm), 0>; 1305 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1306 (!cast<Instruction>(NAME # _S) PNRAny:$Pd, 1307 PNRAny:$Pn, PPR32:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_3:$imm), 0>; 1308 def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", 1309 (!cast<Instruction>(NAME # _D) PNRAny:$Pd, 1310 PNRAny:$Pn, PPR64:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_1:$imm), 0>; 1311 1312 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPR8:$Pm), 1313 MatrixIndexGPR32Op12_15:$idx)), 1314 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, 0)>; 1315 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv8i1 PPR16:$Pm), 1316 MatrixIndexGPR32Op12_15:$idx)), 1317 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, 0)>; 1318 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv4i1 PPR32:$Pm), 1319 MatrixIndexGPR32Op12_15:$idx)), 1320 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, 0)>; 1321 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv2i1 PPR64:$Pm), 1322 MatrixIndexGPR32Op12_15:$idx)), 1323 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, 0)>; 1324 1325 let AddedComplexity = 1 in { 1326 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPR8:$Pm), 1327 (i32 (tileslice8 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm)))), 1328 (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, $imm)>; 1329 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv8i1 PPR16:$Pm), 1330 (i32 (tileslice16 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_7:$imm)))), 1331 (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, $imm)>; 1332 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv4i1 PPR32:$Pm), 1333 (i32 (tileslice32 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_3:$imm)))), 1334 (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, $imm)>; 1335 def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv2i1 PPR64:$Pm), 1336 (i32 (tileslice64 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_1:$imm)))), 1337 (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, $imm)>; 1338 } 1339} 1340 1341//===----------------------------------------------------------------------===// 1342// SME2 Instructions 1343//===----------------------------------------------------------------------===// 1344 1345//===----------------------------------------------------------------------===// 1346// SME2 single-multi ternary int/fp, two/four registers 1347 1348class sme2_dot_mla_add_sub_array_vg24_single<bits<7> op, 1349 MatrixOperand matrix_ty, 1350 RegisterOperand multi_vector_ty, 1351 ZPRRegOp zpr_ty, 1352 string mnemonic> 1353 : I<(outs matrix_ty:$ZAd), 1354 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1355 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 1356 mnemonic,"\t$ZAd[$Rv, $imm3, " # !if(op{5}, "vgx4", "vgx2") # "], $Zn, $Zm", 1357 "", []> , Sched<[]> { 1358 bits<4> Zm; 1359 bits<5> Zn; 1360 bits<2> Rv; 1361 bits<3> imm3; 1362 let Inst{31-23} = 0b110000010; 1363 let Inst{22} = op{6}; //sz 1364 let Inst{21} = 0b1; 1365 let Inst{20} = op{5}; //vgx4 1366 let Inst{19-16} = Zm; 1367 let Inst{15} = 0b0; 1368 let Inst{14-13} = Rv; 1369 let Inst{12-10} = op{4-2}; 1370 let Inst{9-5} = Zn; 1371 let Inst{4-3} = op{1-0}; 1372 let Inst{2-0} = imm3; 1373 let Constraints = "$ZAd = $_ZAd"; 1374} 1375 1376multiclass sme2_dot_mla_add_sub_array_vg24_single<string mnemonic, bits<7> op, 1377 MatrixOperand matrix_ty, 1378 RegisterOperand multi_vector_ty, 1379 ZPRRegOp zpr_ty>{ 1380 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1381 1382 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1383 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1384} 1385 1386multiclass sme2_dot_mla_add_sub_array_vg2_single<string mnemonic, bits<7> op, 1387 MatrixOperand matrix_ty, 1388 RegisterOperand multi_vector_ty, 1389 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ 1390 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1391 1392 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1393 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1394 1395 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; 1396 1397 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; 1398} 1399 1400multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op, 1401 MatrixOperand matrix_ty, 1402 RegisterOperand multi_vector_ty, 1403 ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ 1404 def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1405 1406 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1407 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 1408 1409 def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; 1410 1411 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; 1412} 1413 1414//===----------------------------------------------------------------------===// 1415// SME2 multiple vectors ternary INT/FP two and four registers 1416class sme2_dot_mla_add_sub_array_vg2_multi<bits<6> op, 1417 MatrixOperand matrix_ty, 1418 RegisterOperand multi_vector_ty, 1419 string mnemonic> 1420 : I<(outs matrix_ty:$ZAd), 1421 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1422 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 1423 mnemonic, "\t$ZAd[$Rv, $imm3, vgx2], $Zn, $Zm", 1424 "", []>, Sched<[]>{ 1425 bits<4> Zm; 1426 bits<4> Zn; 1427 bits<2> Rv; 1428 bits<3> imm3; 1429 let Inst{31-23} = 0b110000011; 1430 let Inst{22} = op{5}; //sz 1431 let Inst{21} = 0b1; 1432 let Inst{20-17} = Zm; 1433 let Inst{16-15} = 0b00; 1434 let Inst{14-13} = Rv; 1435 let Inst{12-10} = op{4-2}; 1436 let Inst{9-6} = Zn; 1437 let Inst{5} = 0b0; 1438 let Inst{4-3} = op{1-0}; 1439 let Inst{2-0} = imm3; 1440 let Constraints = "$ZAd = $_ZAd"; 1441} 1442 1443multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<6> op, 1444 MatrixOperand matrix_ty, 1445 RegisterOperand multi_vector_ty, ValueType zpr_ty, 1446 SDPatternOperator intrinsic> { 1447 def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1448 1449 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; 1450 1451 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; 1452 1453 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1454 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 1455} 1456 1457class sme2_dot_mla_add_sub_array_vg4_multi<bits<6> op, 1458 MatrixOperand matrix_ty, 1459 RegisterOperand multi_vector_ty, 1460 string mnemonic> 1461 : I<(outs matrix_ty:$ZAd), 1462 (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, 1463 sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 1464 mnemonic, "\t$ZAd[$Rv, $imm3, vgx4], $Zn, $Zm", 1465 "", []>, Sched<[]>{ 1466 bits<3> Zm; 1467 bits<3> Zn; 1468 bits<2> Rv; 1469 bits<3> imm3; 1470 let Inst{31-23} = 0b110000011; 1471 let Inst{22} = op{5}; //sz 1472 let Inst{21} = 0b1; 1473 let Inst{20-18} = Zm; 1474 let Inst{17-15} = 0b010; 1475 let Inst{14-13} = Rv; 1476 let Inst{12-10} = op{4-2}; 1477 let Inst{9-7} = Zn; 1478 let Inst{6-5} = 0b00; 1479 let Inst{4-3} = op{1-0}; 1480 let Inst{2-0} = imm3; 1481 let Constraints = "$ZAd = $_ZAd"; 1482} 1483 1484multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<6> op, 1485 MatrixOperand matrix_ty, 1486 RegisterOperand multi_vector_ty, 1487 ValueType zpr_ty, SDPatternOperator intrinsic>{ 1488 def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 1489 1490 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; 1491 1492 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; 1493 1494 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", 1495 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; 1496} 1497 1498//===----------------------------------------------------------------------===// 1499// SME2 multiple vectors binary two or four registers 1500 1501class sme2_multivec_accum_add_sub<string mnemonic, bit sz, bit vg4, bits<3> op, 1502 MatrixOperand matrix_ty, 1503 RegisterOperand vector_ty> 1504 : I<(outs matrix_ty:$ZAdn), 1505 (ins matrix_ty:$_ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 1506 mnemonic, "\t$ZAdn[$Rv, $imm3, " # !if(vg4, "vgx4", "vgx2") # "], $Zm", 1507 "", []>, Sched<[]> { 1508 bits<2> Rv; 1509 bits<3> imm3; 1510 let Inst{31-23} = 0b110000011; 1511 let Inst{22} = sz; 1512 let Inst{21-19} = 0b100; 1513 let Inst{18} = op{2}; 1514 let Inst{17} = 0b0; 1515 let Inst{16} = vg4; 1516 let Inst{15} = 0b0; 1517 let Inst{14-13} = Rv; 1518 let Inst{12-10} = 0b111; 1519 let Inst{5} = 0b0; 1520 let Inst{4-3} = op{1-0}; 1521 let Inst{2-0} = imm3; 1522 1523 let Constraints = "$ZAdn = $_ZAdn"; 1524} 1525 1526class sme2_multivec_accum_add_sub_vg2<string mnemonic, bit sz, bits<3> op, 1527 MatrixOperand matrix_ty, 1528 RegisterOperand vector_ty> 1529 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b0, op, matrix_ty, vector_ty> { 1530 bits<4> Zm; 1531 let Inst{9-6} = Zm; 1532} 1533 1534 1535multiclass sme2_multivec_accum_add_sub_vg2<string mnemonic, bits<4> op, 1536 MatrixOperand matrix_ty, 1537 RegisterOperand vector_ty, 1538 ValueType vty, 1539 SDPatternOperator intrinsic> { 1540 def NAME : sme2_multivec_accum_add_sub_vg2<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>, 1541 SMEPseudo2Instr<NAME, 1>; 1542 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", 1543 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; 1544 1545 def _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, vector_ty, SMEMatrixArray>; 1546 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, vty, sme_elm_idx0_7, tileslice16>; 1547} 1548 1549class sme2_multivec_accum_add_sub_vg4<string mnemonic, bit sz, bits<3> op, 1550 MatrixOperand matrix_ty, 1551 RegisterOperand vector_ty> 1552 : sme2_multivec_accum_add_sub<mnemonic, sz, 0b1, op, matrix_ty, vector_ty> { 1553 bits<3> Zm; 1554 let Inst{9-7} = Zm; 1555 let Inst{6} = 0b0; 1556} 1557 1558multiclass sme2_multivec_accum_add_sub_vg4<string mnemonic, bits<4> op, 1559 MatrixOperand matrix_ty, 1560 RegisterOperand vector_ty, 1561 ValueType vty, 1562 SDPatternOperator intrinsic> { 1563 def NAME : sme2_multivec_accum_add_sub_vg4<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>, 1564 SMEPseudo2Instr<NAME, 1>; 1565 def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", 1566 (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; 1567 1568 def _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, vector_ty, SMEMatrixArray>; 1569 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, vty, sme_elm_idx0_7, tileslice16>; 1570} 1571 1572//===----------------------------------------------------------------------===// 1573// SME2 Multi-vector - Multiple and Single SVE Destructive 1574// Two and Four registers 1575 1576class sme2_sve_destructive_vector_vg2_single<bits<2> sz, bits<7> op, 1577 RegisterOperand vector_ty, 1578 ZPRRegOp zpr_ty, 1579 string mnemonic> 1580 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), 1581 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1582 "", []>, Sched<[]> { 1583 bits<4> Zm; 1584 bits<4> Zdn; 1585 let Inst{31-24} = 0b11000001; 1586 let Inst{23-22} = sz; 1587 let Inst{21-20} = 0b10; 1588 let Inst{19-16} = Zm; 1589 let Inst{15-11} = 0b10100; 1590 let Inst{10-5} = op{6-1}; 1591 let Inst{4-1} = Zdn; 1592 let Inst{0} = op{0}; 1593 1594 let Constraints = "$Zdn = $_Zdn"; 1595} 1596 1597multiclass sme2_fp_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { 1598 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; 1599 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; 1600 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; 1601} 1602 1603multiclass sme2_int_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { 1604 def _B : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_b_mul_r, ZPR4b8, mnemonic>; 1605 def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; 1606 def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; 1607 def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; 1608} 1609 1610// SME2.1 fmax/fmin instructions. 1611multiclass sme2p1_bf_max_min_vector_vg2_single<string mnemonic, bits<7>op> { 1612 def _H : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_h_mul_r, 1613 ZPR4b16, mnemonic>; 1614} 1615 1616class sme2_sve_destructive_vector_vg4_single<bits<2> sz, bits<7> op, 1617 RegisterOperand vector_ty, 1618 ZPRRegOp zpr_ty, 1619 string mnemonic> 1620 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), 1621 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1622 "", []>, Sched<[]> { 1623 bits<4> Zm; 1624 bits<3> Zdn; 1625 let Inst{31-24} = 0b11000001; 1626 let Inst{23-22} = sz; 1627 let Inst{21-20} = 0b10; 1628 let Inst{19-16} = Zm; 1629 let Inst{15-11} = 0b10101; 1630 let Inst{10-5} = op{6-1}; 1631 let Inst{4-2} = Zdn; 1632 let Inst{1} = 0b0; 1633 let Inst{0} = op{0}; 1634 1635 let Constraints = "$Zdn = $_Zdn"; 1636} 1637 1638multiclass sme2_fp_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { 1639 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; 1640 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; 1641 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; 1642} 1643 1644multiclass sme2_int_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { 1645 def _B : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_b_mul_r, ZPR4b8, mnemonic>; 1646 def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; 1647 def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; 1648 def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; 1649} 1650 1651// SME2.1 fmax/fmin instructions. 1652multiclass sme2p1_bf_max_min_vector_vg4_single<string mnemonic, bits<7>op> { 1653 def _H : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_h_mul_r, 1654 ZPR4b16, mnemonic>; 1655} 1656 1657class sme2_sve_destructive_vector_vg2_multi<bits<2> sz, bits<7> op, 1658 RegisterOperand vector_ty, 1659 string mnemonic> 1660 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), 1661 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1662 "", []>, Sched<[]> { 1663 bits<4> Zm; 1664 bits<4> Zdn; 1665 let Inst{31-24} = 0b11000001; 1666 let Inst{23-22} = sz; 1667 let Inst{21} = 0b1; 1668 let Inst{20-17} = Zm; 1669 let Inst{16-11} = 0b010110; 1670 let Inst{10-5} = op{6-1}; 1671 let Inst{4-1} = Zdn; 1672 let Inst{0} = op{0}; 1673 1674 let Constraints = "$Zdn = $_Zdn"; 1675} 1676 1677multiclass sme2_fp_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { 1678 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; 1679 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; 1680 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; 1681} 1682 1683multiclass sme2_int_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { 1684 def _B : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_b_mul_r, mnemonic>; 1685 def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; 1686 def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; 1687 def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; 1688} 1689 1690// SME2.1 fmax/fmin instructions. 1691multiclass sme2p1_bf_max_min_vector_vg2_multi<string mnemonic, bits<7>op> { 1692 def _H : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_h_mul_r, 1693 mnemonic>; 1694} 1695 1696class sme2_sve_destructive_vector_vg4_multi<bits<2> sz, bits<7> op, 1697 RegisterOperand vector_ty, 1698 string mnemonic> 1699 : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), 1700 mnemonic, "\t$Zdn, $_Zdn, $Zm", 1701 "", []>, Sched<[]> { 1702 bits<3> Zm; 1703 bits<3> Zdn; 1704 let Inst{31-24} = 0b11000001; 1705 let Inst{23-22} = sz; 1706 let Inst{21} = 0b1; 1707 let Inst{20-18} = Zm; 1708 let Inst{17-11} = 0b0010111; 1709 let Inst{10-5} = op{6-1}; 1710 let Inst{4-2} = Zdn; 1711 let Inst{1} = 0b0; 1712 let Inst{0} = op{0}; 1713 1714 let Constraints = "$Zdn = $_Zdn"; 1715} 1716 1717multiclass sme2_fp_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { 1718 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; 1719 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; 1720 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; 1721} 1722 1723multiclass sme2_int_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { 1724 def _B : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_b_mul_r, mnemonic>; 1725 def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; 1726 def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; 1727 def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; 1728} 1729 1730// SME2.1 fmax/fmin instructions. 1731multiclass sme2p1_bf_max_min_vector_vg4_multi<string mnemonic, bits<7>op> { 1732 def _H : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_h_mul_r, 1733 mnemonic>; 1734} 1735 1736//===----------------------------------------------------------------------===// 1737// SME2 Multi-vector - Index/Single/Multi Array Vectors FMA sources 1738 1739class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty, 1740 RegisterOperand multi_vector_ty, 1741 string mnemonic, string vg_acronym=""> 1742 : I<(outs MatrixOp32:$ZAda), 1743 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm, multi_vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 1744 mnemonic, "\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm$i3", 1745 "", []>, Sched<[]> { 1746 bits<4> Zm; 1747 bits<2> Rv; 1748 let Inst{31-24} = 0b11000001; 1749 let Inst{23-22} = op0; 1750 let Inst{21} = 0b0; 1751 let Inst{20} = !if(!eq(vg_acronym, ""), 0, 1); 1752 let Inst{19-16} = Zm; 1753 let Inst{14-13} = Rv; 1754 let Inst{12} = 0b1; 1755 let Inst{4-3} = op; 1756 1757 let Constraints = "$ZAda = $_ZAda"; 1758} 1759 1760multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1761 def _S : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16, 1762 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { 1763 bits<3> i3; 1764 bits<5> Zn; 1765 bits<3> imm; 1766 let Inst{15} = i3{2}; 1767 let Inst{11-10} = i3{1-0}; 1768 let Inst{9-5} = Zn; 1769 let Inst{2-0} = imm; 1770 } 1771 1772 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1773 1774 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>; 1775} 1776 1777class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op> 1778 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZ_h_mul_r, 1779 mnemonic, "vgx2"> { 1780 bits<3> i3; 1781 bits<4> Zn; 1782 bits<2> imm; 1783 let Inst{15} = 0b0; 1784 let Inst{11-10} = i3{2-1}; 1785 let Inst{9-6} = Zn; 1786 let Inst{5} = 0b0; 1787 let Inst{2} = i3{0}; 1788 let Inst{1-0} = imm; 1789} 1790 1791multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1792 def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1793 1794 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1795 1796 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; 1797 1798 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1799 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1800} 1801 1802multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1803 def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1804 1805 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1806 1807 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; 1808 1809 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1810 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1811} 1812 1813class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op> 1814 : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZZZ_h_mul_r, 1815 mnemonic, "vgx4"> { 1816 bits<3> i3; 1817 bits<3> Zn; 1818 bits<2> imm; 1819 let Inst{15} = 0b1; 1820 let Inst{11-10} = i3{2-1}; 1821 let Inst{9-7} = Zn; 1822 let Inst{6-5} = 0b00; 1823 let Inst{2} = i3{0}; 1824 let Inst{1-0} = imm; 1825} 1826 1827multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1828 def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1829 1830 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1831 1832 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; 1833 1834 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1835 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1836} 1837 1838multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1839 def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1840 1841 def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 1842 1843 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; 1844 1845 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", 1846 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; 1847} 1848 1849class sme2_mla_long_array<bits<2>op0, bits<2> op, Operand index_ty, 1850 RegisterOperand first_vector_ty, 1851 RegisterOperand second_vector_ty, 1852 string mnemonic, string vg_acronym=""> 1853 : I<(outs MatrixOp32:$ZAda), 1854 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, 1855 index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm), 1856 mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm", 1857 "", []> , Sched<[]> { 1858 bits<2> Rv; 1859 let Inst{31-24} = 0b11000001; 1860 let Inst{23-22} = op0; 1861 let Inst{21} = 0b1; 1862 let Inst{15} = 0b0; 1863 let Inst{14-13} = Rv; 1864 let Inst{12-11} = 0b01; 1865 let Inst{10} = !if(!eq(vg_acronym, ""), 1, 0); 1866 let Inst{4-3} = op; 1867 1868 let Constraints = "$ZAda = $_ZAda"; 1869} 1870 1871multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1872 def _S : sme2_mla_long_array<op0, op, uimm3s2range, ZPR16, ZPR4b16, 1873 mnemonic> , SMEPseudo2Instr<NAME # _S, 1>{ 1874 bits<4> Zm; 1875 bits<5> Zn; 1876 bits<3> imm; 1877 let Inst{20} = 0b0; 1878 let Inst{19-16} = Zm; 1879 let Inst{9-5} = Zn; 1880 let Inst{2-0} = imm; 1881 } 1882 1883 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>; 1884 1885 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>; 1886} 1887 1888class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op, 1889 RegisterOperand first_vector_ty, 1890 string mnemonic, string vg_acronym> 1891 : sme2_mla_long_array<op0, op, uimm2s2range, first_vector_ty, ZPR4b16, 1892 mnemonic, vg_acronym> { 1893 bits<4> Zm; 1894 bits<5> Zn; 1895 bits<2> imm; 1896 let Inst{20} = vg4; 1897 let Inst{19-16} = Zm; 1898 let Inst{9-5} = Zn; 1899 let Inst{2} = 0b0; 1900 let Inst{1-0} = imm; 1901} 1902 1903multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1904 def _S : sme2_mla_long_array_vg24_single<0b00, 0b0, op, ZZ_h, mnemonic, 1905 "vgx2">, SMEPseudo2Instr<NAME # _S, 1>; 1906 1907 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; 1908 1909 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>; 1910 1911 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1912 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1913} 1914 1915multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1916 def _S : sme2_mla_long_array_vg24_single<0b01, 0b0, op, ZZ_h, mnemonic, 1917 "vgx2">, SMEPseudo2Instr<NAME # _S, 1>; 1918 1919 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; 1920 1921 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; 1922 1923 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1924 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1925} 1926 1927multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1928 def _S : sme2_mla_long_array_vg24_single<0b00, 0b1, op, ZZZZ_h, mnemonic, 1929 "vgx4">, SMEPseudo2Instr<NAME # _S, 1>; 1930 1931 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; 1932 1933 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>; 1934 1935 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1936 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1937} 1938 1939multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1940 def _S : sme2_mla_long_array_vg24_single<0b01, 0b1, op, ZZZZ_h, mnemonic, 1941 "vgx4">, SMEPseudo2Instr<NAME # _S, 1>; 1942 1943 def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; 1944 1945 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; 1946 1947 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1948 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; 1949} 1950class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<2> op> 1951 : sme2_mla_long_array<op0, op, uimm2s2range, ZZ_h_mul_r, ZZ_h_mul_r, mnemonic, 1952 "vgx2"> { 1953 bits<4> Zm; 1954 bits<4> Zn; 1955 bits<2> imm; 1956 let Inst{20-17} = Zm; 1957 let Inst{16} = 0b0; 1958 let Inst{9-6} = Zn; 1959 let Inst{5} = 0b0; 1960 let Inst{2} = 0b0; 1961 let Inst{1-0} = imm; 1962} 1963 1964multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 1965 def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 1966 1967 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; 1968 1969 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; 1970 1971 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 1972 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; 1973} 1974 1975multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 1976 def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 1977 1978 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; 1979 1980 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; 1981 1982 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", 1983 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; 1984} 1985 1986class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<2> op> 1987 : sme2_mla_long_array<op0, op, uimm2s2range, ZZZZ_h_mul_r, ZZZZ_h_mul_r, mnemonic, 1988 "vgx4"> { 1989 bits<3> Zm; 1990 bits<3> Zn; 1991 bits<2> imm; 1992 let Inst{20-18} = Zm; 1993 let Inst{17} = 0b0; 1994 let Inst{16} = 0b1; 1995 let Inst{9-7} = Zn; 1996 let Inst{6-5} = 0b00; 1997 let Inst{2} = 0b0; 1998 let Inst{1-0} = imm; 1999} 2000 2001multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { 2002 def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; 2003 2004 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; 2005 2006 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; 2007 2008 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2009 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; 2010} 2011 2012multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2013 def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; 2014 2015 def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; 2016 2017 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; 2018 2019 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", 2020 (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; 2021} 2022 2023//===----------------------------------------------------------------------===// 2024class sme2_frint_cvt_vg2_multi<bits<2>sz, bits<5>op, RegisterOperand first_ty, 2025 RegisterOperand second_ty, string mnemonic> 2026 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2027 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2028 bits<4> Zn; 2029 bits<4> Zd; 2030 let Inst{31-24} = 0b11000001; 2031 let Inst{23-22} = sz; 2032 let Inst{21-20} = 0b10; 2033 let Inst{19-16} = op{4-1}; 2034 let Inst{15-10} = 0b111000; 2035 let Inst{9-6} = Zn; 2036 let Inst{5} = op{0}; 2037 let Inst{4-1} = Zd; 2038 let Inst{0} = 0b0; 2039} 2040 2041// SME2 multi-vec FP to int convert two registers 2042// SME2 multi-vec int to FP two registers 2043multiclass sme2_fp_cvt_vg2_multi<string mnemonic, bits<5> op> { 2044 def NAME : sme2_frint_cvt_vg2_multi<0b00, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; 2045} 2046 2047// SME2 multi-vec FRINT two registers 2048multiclass sme2_frint_vector_vg2_multi<string mnemonic, bits<5> op> { 2049 def _S : sme2_frint_cvt_vg2_multi<0b10, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; 2050} 2051 2052class sme2_frint_zip_cvt_vg4_multi<bits<2>sz, bits<7>op, RegisterOperand first_ty, 2053 RegisterOperand second_ty, string mnemonic> 2054 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2055 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2056 bits<3> Zn; 2057 bits<3> Zd; 2058 let Inst{31-24} = 0b11000001; 2059 let Inst{23-22} = sz; 2060 let Inst{21-20} = 0b11; 2061 let Inst{19-16} = op{6-3}; 2062 let Inst{15-10} = 0b111000; 2063 let Inst{9-7} = Zn; 2064 let Inst{6-5} = op{2-1}; 2065 let Inst{4-2} = Zd; 2066 let Inst{1} = op{0}; 2067 let Inst{0} = 0b0; 2068} 2069 2070// SME2 multi-vec FP to int convert four registers 2071// SME2 multi-vec int to FP four registers 2072multiclass sme2_fp_cvt_vg4_multi<string mnemonic, bits<7> op> { 2073 def NAME : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, mnemonic>; 2074} 2075 2076// SME2 multi-vec quadwords ZIP four registers 2077multiclass sme2_zip_vector_vg4<string mnemonic, bits<7> op> { 2078 def _B : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_b_mul_r, ZZZZ_b_mul_r, 2079 mnemonic>; 2080 def _H : sme2_frint_zip_cvt_vg4_multi<0b01, op, ZZZZ_h_mul_r, ZZZZ_h_mul_r, 2081 mnemonic>; 2082 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, 2083 mnemonic>; 2084 def _D : sme2_frint_zip_cvt_vg4_multi<0b11, op, ZZZZ_d_mul_r, ZZZZ_d_mul_r, 2085 mnemonic>; 2086} 2087 2088// SME2 multi-vec quadwords ZIP four registers 2089multiclass sme2_zip_vector_vg4_Q<string mnemonic, bits<7> op> { 2090 def NAME: sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_q_mul_r, ZZZZ_q_mul_r, 2091 mnemonic>; 2092} 2093 2094// SME2 multi-vec FRINT four registers 2095multiclass sme2_frint_vector_vg4_multi<string mnemonic, bits<7> op> { 2096 def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, 2097 mnemonic>; 2098} 2099 2100class sme2_cvt_vg2_single<string mnemonic, bits<4> op> 2101 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn), 2102 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2103 bits<4> Zn; 2104 bits<5> Zd; 2105 let Inst{31-23} = 0b110000010; 2106 let Inst{22} = op{3}; 2107 let Inst{21-18} = 0b1000; 2108 let Inst{17-16} = op{2-1}; 2109 let Inst{15-10} = 0b111000; 2110 let Inst{9-6} = Zn; 2111 let Inst{5} = op{0}; 2112 let Inst{4-0} = Zd; 2113} 2114 2115// SME2 multi-vec FP down convert two registers 2116// SME2 multi-vec int down convert two registers 2117multiclass sme2_cvt_vg2_single<string mnemonic, bits<4> op, ValueType out_vt, 2118 ValueType in_vt, SDPatternOperator intrinsic> { 2119 def NAME : sme2_cvt_vg2_single<mnemonic, op>; 2120 def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, out_vt, in_vt>; 2121} 2122 2123class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty, 2124 RegisterOperand second_ty, string mnemonic> 2125 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2126 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2127 bits<5> Zn; 2128 bits<4> Zd; 2129 let Inst{31-24} = 0b11000001; 2130 let Inst{23-22} = sz; 2131 let Inst{21-19} = 0b100; 2132 let Inst{18-16} = op; 2133 let Inst{15-10} = 0b111000; 2134 let Inst{9-5} = Zn; 2135 let Inst{4-1} = Zd; 2136 let Inst{0} = u; 2137} 2138 2139// SME2 multi-vec unpack two registers 2140multiclass sme2_unpk_vector_vg2<string mnemonic, bit u> { 2141 def _H : sme2_cvt_unpk_vector_vg2<0b01, 0b101, u, ZZ_h_mul_r, ZPR8, mnemonic>; 2142 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b101, u, ZZ_s_mul_r, ZPR16, mnemonic>; 2143 def _D : sme2_cvt_unpk_vector_vg2<0b11, 0b101, u, ZZ_d_mul_r, ZPR32, mnemonic>; 2144} 2145 2146// SME2.1 multi-vec convert two registers 2147multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> { 2148 def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b000, l, ZZ_s_mul_r, ZPR16, mnemonic>; 2149} 2150 2151class sme2_cvt_vg4_single<bit sz, bits<3> op, RegisterOperand first_ty, 2152 RegisterOperand second_ty, string mnemonic> 2153 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2154 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2155 bits<3> Zn; 2156 bits<5> Zd; 2157 let Inst{31-24} = 0b11000001; 2158 let Inst{23} = sz; 2159 let Inst{22} = op{2}; 2160 let Inst{21-10} = 0b110011111000; 2161 let Inst{9-7} = Zn; 2162 let Inst{6-5} = op{1-0}; 2163 let Inst{4-0} = Zd; 2164} 2165 2166// SME2 multi-vec int down convert four registers 2167multiclass sme2_int_cvt_vg4_single<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2168 def _StoB : sme2_cvt_vg4_single<0, op, ZPR8, ZZZZ_s_mul_r, mnemonic>; 2169 def _DtoH : sme2_cvt_vg4_single<1, op, ZPR16, ZZZZ_d_mul_r, mnemonic>; 2170 2171 def : SME2_Cvt_VG4_Pat<NAME # _StoB, intrinsic, nxv16i8, nxv4i32>; 2172 def : SME2_Cvt_VG4_Pat<NAME # _DtoH, intrinsic, nxv8i16, nxv2i64>; 2173} 2174 2175class sme2_unpk_vector_vg4<bits<2>sz, bit u, RegisterOperand first_ty, 2176 RegisterOperand second_ty, string mnemonic> 2177 : I<(outs first_ty:$Zd), (ins second_ty:$Zn), 2178 mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { 2179 bits<4> Zn; 2180 bits<3> Zd; 2181 let Inst{31-24} = 0b11000001; 2182 let Inst{23-22} = sz; 2183 let Inst{21-10} = 0b110101111000; 2184 let Inst{9-6} = Zn; 2185 let Inst{5} = 0b0; 2186 let Inst{4-2} = Zd; 2187 let Inst{1} = 0b0; 2188 let Inst{0} = u; 2189} 2190 2191// SME2 multi-vec UNPK four registers 2192multiclass sme2_unpk_vector_vg4<string mnemonic, bit u> { 2193 def _H : sme2_unpk_vector_vg4<0b01, u, ZZZZ_h_mul_r, ZZ_b_mul_r, mnemonic>; 2194 def _S : sme2_unpk_vector_vg4<0b10, u, ZZZZ_s_mul_r, ZZ_h_mul_r, mnemonic>; 2195 def _D : sme2_unpk_vector_vg4<0b11, u, ZZZZ_d_mul_r, ZZ_s_mul_r, mnemonic>; 2196} 2197 2198//===----------------------------------------------------------------------===// 2199// SME2 multi-vec CLAMP registers 2200 2201class sme2_clamp_vector_vg24_multi<bits<2> sz, bits<3> op1, bit u, 2202 RegisterOperand multi_vector_ty, 2203 ZPRRegOp vector_ty, string mnemonic> 2204 : I<(outs multi_vector_ty:$Zd), 2205 (ins multi_vector_ty:$_Zd, vector_ty:$Zn, vector_ty:$Zm), 2206 mnemonic, "\t$Zd, $Zn, $Zm", 2207 "", []>, Sched<[]>{ 2208 bits<5> Zm; 2209 bits<5> Zn; 2210 let Inst{31-24} = 0b11000001; 2211 let Inst{23-22} = sz; 2212 let Inst{21} = 0b1; 2213 let Inst{20-16} = Zm; 2214 let Inst{15-13} = 0b110; 2215 let Inst{12-10} = op1; 2216 let Inst{9-5} = Zn; 2217 let Inst{0} = u; 2218 2219 let Constraints = "$Zd = $_Zd"; 2220} 2221 2222class sme2_clamp_vector_vg2_multi<bits<2> sz, bits<3> op1, bit u, 2223 RegisterOperand multi_vector_ty, 2224 ZPRRegOp vector_ty, string mnemonic> 2225 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, 2226 mnemonic>{ 2227 bits<4> Zd; 2228 let Inst{4-1} = Zd; 2229} 2230 2231multiclass sme2_fp_clamp_vector_vg2_multi<string mnemonic>{ 2232 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b000, 0b0, ZZ_h_mul_r, ZPR16, mnemonic>; 2233 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b000, 0b0, ZZ_s_mul_r, ZPR32, mnemonic>; 2234 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b000, 0b0, ZZ_d_mul_r, ZPR64, mnemonic>; 2235} 2236 2237multiclass sme2_int_clamp_vector_vg2_multi<string mnemonic, bit u>{ 2238 def _B : sme2_clamp_vector_vg2_multi<0b00, 0b001, u, ZZ_b_mul_r, ZPR8, mnemonic>; 2239 def _H : sme2_clamp_vector_vg2_multi<0b01, 0b001, u, ZZ_h_mul_r, ZPR16, mnemonic>; 2240 def _S : sme2_clamp_vector_vg2_multi<0b10, 0b001, u, ZZ_s_mul_r, ZPR32, mnemonic>; 2241 def _D : sme2_clamp_vector_vg2_multi<0b11, 0b001, u, ZZ_d_mul_r, ZPR64, mnemonic>; 2242} 2243 2244// SME2.1 multi-vec FCLAMP two registers 2245multiclass sme2p1_bfclamp_vector_vg2_multi<string mnemonic> { 2246 def _H : sme2_clamp_vector_vg2_multi<0b00, 0b000, 0b0, ZZ_h_mul_r, ZPR16, 2247 mnemonic>; 2248} 2249 2250class sme2_clamp_vector_vg4_multi<bits<2> sz, bits<3> op1, bit u, 2251 RegisterOperand multi_vector_ty, 2252 ZPRRegOp vector_ty, string mnemonic> 2253 : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, 2254 mnemonic>{ 2255 bits<3> Zd; 2256 let Inst{4-2} = Zd; 2257 let Inst{1} = 0b0; 2258} 2259 2260multiclass sme2_fp_clamp_vector_vg4_multi<string mnemonic>{ 2261 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, mnemonic>; 2262 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b010, 0b0, ZZZZ_s_mul_r, ZPR32, mnemonic>; 2263 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b010, 0b0, ZZZZ_d_mul_r, ZPR64, mnemonic>; 2264} 2265 2266multiclass sme2_int_clamp_vector_vg4_multi<string mnemonic, bit u>{ 2267 def _B : sme2_clamp_vector_vg4_multi<0b00, 0b011, u, ZZZZ_b_mul_r, ZPR8, mnemonic>; 2268 def _H : sme2_clamp_vector_vg4_multi<0b01, 0b011, u, ZZZZ_h_mul_r, ZPR16, mnemonic>; 2269 def _S : sme2_clamp_vector_vg4_multi<0b10, 0b011, u, ZZZZ_s_mul_r, ZPR32, mnemonic>; 2270 def _D : sme2_clamp_vector_vg4_multi<0b11, 0b011, u, ZZZZ_d_mul_r, ZPR64, mnemonic>; 2271} 2272 2273// SME2.1 multi-vec FCLAMP four registers 2274multiclass sme2p1_bfclamp_vector_vg4_multi<string mnemonic> { 2275 def _H : sme2_clamp_vector_vg4_multi<0b00, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, 2276 mnemonic>; 2277} 2278 2279// SME2 multi-vec ZIP two registers 2280class sme2_zip_vector_vg2<bits<2> sz, bit q, bit u, 2281 RegisterOperand multi_vector_ty, 2282 ZPRRegOp vector_ty, string mnemonic> 2283 : I<(outs multi_vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm), 2284 mnemonic, "\t$Zd, $Zn, $Zm", 2285 "", []>, Sched<[]>{ 2286 bits<4> Zd; 2287 bits<5> Zm; 2288 bits<5> Zn; 2289 let Inst{31-24} = 0b11000001; 2290 let Inst{23-22} = sz; 2291 let Inst{21} = 0b1; 2292 let Inst{20-16} = Zm; 2293 let Inst{15-11} = 0b11010; 2294 let Inst{10} = q; 2295 let Inst{9-5} = Zn; 2296 let Inst{4-1} = Zd; 2297 let Inst{0} = u; 2298} 2299 2300multiclass sme2_zip_vector_vg2<string mnemonic, bit op> { 2301 def _B : sme2_zip_vector_vg2<0b00, 0b0, op, ZZ_b_mul_r, ZPR8, mnemonic>; 2302 def _H : sme2_zip_vector_vg2<0b01, 0b0, op, ZZ_h_mul_r, ZPR16, mnemonic>; 2303 def _S : sme2_zip_vector_vg2<0b10, 0b0, op, ZZ_s_mul_r, ZPR32, mnemonic>; 2304 def _D : sme2_zip_vector_vg2<0b11, 0b0, op, ZZ_d_mul_r, ZPR64, mnemonic>; 2305 def _Q : sme2_zip_vector_vg2<0b00, 0b1, op, ZZ_q_mul_r, ZPR128, mnemonic>; 2306} 2307 2308//===----------------------------------------------------------------------===// 2309// SME2 Dot Products and MLA 2310 2311class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty, 2312 RegisterOperand multi_vector_ty, 2313 ZPRRegOp vector_ty, Operand index_ty, 2314 string mnemonic> 2315 : I<(outs matrix_ty:$ZAda), 2316 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2317 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), 2318 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i", 2319 "", []>, Sched<[]> { 2320 bits<4> Zm; 2321 bits<2> Rv; 2322 bits<4> Zn; 2323 bits<3> imm3; 2324 let Inst{31-23} = 0b110000010; 2325 let Inst{22} = sz; 2326 let Inst{21-20} = 0b01; 2327 let Inst{19-16} = Zm; 2328 let Inst{15} = 0b0; 2329 let Inst{14-13} = Rv; 2330 let Inst{12-10} = op{5-3}; 2331 let Inst{9-6} = Zn; 2332 let Inst{5-3} = op{2-0}; 2333 let Inst{2-0} = imm3; 2334 2335 let Constraints = "$ZAda = $_ZAda"; 2336} 2337 2338// SME2 multi-vec ternary indexed two registers 32-bit 2339multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<4> op, 2340 RegisterOperand multi_vector_ty, 2341 ZPRRegOp vector_ty, ValueType vt, 2342 SDPatternOperator intrinsic> { 2343 def NAME : sme2_multi_vec_array_vg2_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty, 2344 VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2345 bits<2> i; 2346 let Inst{11-10} = i; 2347 } 2348 2349 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; 2350 2351 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; 2352 2353 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2354 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2355 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; 2356} 2357 2358// SME2.1 multi-vec ternary indexed two registers 16-bit 2359multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> op> { 2360 def NAME : sme2_multi_vec_array_vg2_index<0b0, {0b1,?,?,op,?}, MatrixOp16, 2361 ZZ_h_mul_r, ZPR4b16, 2362 VectorIndexH, mnemonic> { 2363 bits<3> i; 2364 let Inst{11-10} = i{2-1}; 2365 let Inst{3} = i{0}; 2366 } 2367 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2368 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2369 ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; 2370} 2371 2372// SME2 multi-vec ternary indexed two registers 64-bit 2373 2374class sme2_multi_vec_array_vg2_index_64b<bits<2> op, 2375 RegisterOperand multi_vector_ty, 2376 ZPRRegOp vector_ty, 2377 string mnemonic> 2378 : I<(outs MatrixOp64:$ZAda), 2379 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2380 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 2381 mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i1", 2382 "", []>, Sched<[]> { 2383 bits<4> Zm; 2384 bits<2> Rv; 2385 bits<1> i1; 2386 bits<4> Zn; 2387 bits<3> imm3; 2388 let Inst{31-20} = 0b110000011101; 2389 let Inst{19-16} = Zm; 2390 let Inst{15} = 0b0; 2391 let Inst{14-13} = Rv; 2392 let Inst{12-11} = 0b00; 2393 let Inst{10} = i1; 2394 let Inst{9-6} = Zn; 2395 let Inst{5} = 0b0; 2396 let Inst{4-3} = op; 2397 let Inst{2-0} = imm3; 2398 2399 let Constraints = "$ZAda = $_ZAda"; 2400} 2401 2402multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op, 2403 RegisterOperand multi_vector_ty, 2404 ZPRRegOp vector_ty, ValueType vt, 2405 SDPatternOperator intrinsic> { 2406 def NAME : sme2_multi_vec_array_vg2_index_64b<op, multi_vector_ty, vector_ty, 2407 mnemonic>, SMEPseudo2Instr<NAME, 1>; 2408 2409 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; 2410 2411 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexD32b_timm, tileslice16>; 2412 2413 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", 2414 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2415 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; 2416} 2417 2418class sme2_multi_vec_array_vg4_index<bit sz, bits<6> op, MatrixOperand matrix_ty, 2419 RegisterOperand multi_vector_ty, 2420 ZPRRegOp vector_ty, Operand index_ty, 2421 string mnemonic> 2422 : I<(outs matrix_ty:$ZAda), 2423 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2424 multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), 2425 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i", 2426 "", []>, Sched<[]> { 2427 bits<4> Zm; 2428 bits<2> Rv; 2429 bits<3> Zn; 2430 bits<3> imm3; 2431 let Inst{31-23} = 0b110000010; 2432 let Inst{22} = sz; 2433 let Inst{21-20} = 0b01; 2434 let Inst{19-16} = Zm; 2435 let Inst{15} = 0b1; 2436 let Inst{14-13} = Rv; 2437 let Inst{12-10} = op{5-3}; 2438 let Inst{9-7} = Zn; 2439 let Inst{6} = 0b0; 2440 let Inst{5-3} = op{2-0}; 2441 let Inst{2-0} = imm3; 2442 2443 let Constraints = "$ZAda = $_ZAda"; 2444} 2445 2446// SME2 multi-vec ternary indexed four registers 32-bit 2447multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op, 2448 RegisterOperand multi_vector_ty, 2449 ZPRRegOp vector_ty, ValueType vt, 2450 SDPatternOperator intrinsic> { 2451 def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, 2452 vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2453 bits<2> i; 2454 let Inst{11-10} = i; 2455 } 2456 2457 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; 2458 2459 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; 2460 2461 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2462 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2463 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; 2464} 2465 2466// SME2.1 multi-vec ternary indexed four registers 16-bit 2467multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<2> op> { 2468 def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16, 2469 ZZZZ_h_mul_r, ZPR4b16, 2470 VectorIndexH, mnemonic>{ 2471 bits<3> i; 2472 let Inst{11-10} = i{2-1}; 2473 let Inst{3} = i{0}; 2474 } 2475 2476 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", 2477 (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, 2478 sme_elm_idx0_7:$imm3, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; 2479} 2480 2481// SME2 multi-vec ternary indexed four registers 64-bit 2482class sme2_multi_vec_array_vg4_index_64b<bits<3> op, 2483 RegisterOperand multi_vector_ty, 2484 ZPRRegOp vector_ty, 2485 string mnemonic> 2486 : I<(outs MatrixOp64:$ZAda), 2487 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2488 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 2489 mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i1", 2490 "", []>, Sched<[]> { 2491 bits<4> Zm; 2492 bits<2> Rv; 2493 bits<1> i1; 2494 bits<3> Zn; 2495 bits<3> imm3; 2496 let Inst{31-20} = 0b110000011101; 2497 let Inst{19-16} = Zm; 2498 let Inst{15} = 0b1; 2499 let Inst{14-13} = Rv; 2500 let Inst{12} = 0b0; 2501 let Inst{11} = op{2}; 2502 let Inst{10} = i1; 2503 let Inst{9-7} = Zn; 2504 let Inst{6-5} = 0b00; 2505 let Inst{4-3} = op{1-0}; 2506 let Inst{2-0} = imm3; 2507 2508 let Constraints = "$ZAda = $_ZAda"; 2509} 2510 2511multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op, 2512 RegisterOperand multi_vector_ty, 2513 ZPRRegOp vector_ty, ValueType vty, 2514 SDPatternOperator intrinsic> { 2515 def NAME : sme2_multi_vec_array_vg4_index_64b<op, multi_vector_ty, vector_ty, 2516 mnemonic>, SMEPseudo2Instr<NAME, 1>; 2517 2518 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; 2519 2520 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vty, VectorIndexD32b_timm, tileslice16>; 2521 2522 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", 2523 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, 2524 multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; 2525} 2526//===----------------------------------------------------------------------===// 2527// SME2 multi-vec indexed long long MLA one source 32-bit 2528class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op> 2529 : I<(outs MatrixOp32:$ZAda), 2530 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 2531 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2532 "", []>, Sched<[]> { 2533 bits<4> Zm; 2534 bits<2> Rv; 2535 bits<4> i; 2536 bits<5> Zn; 2537 bits<2> imm2; 2538 let Inst{31-20} = 0b110000010000; 2539 let Inst{19-16} = Zm; 2540 let Inst{15} = i{3}; 2541 let Inst{14-13} = Rv; 2542 let Inst{12-10} = i{2-0}; 2543 let Inst{9-5} = Zn; 2544 let Inst{4-2} = op; 2545 let Inst{1-0} = imm2; 2546 2547 let Constraints = "$ZAda = $_ZAda"; 2548} 2549 2550multiclass sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2551 def NAME : sme2_mla_ll_array_index_32b<mnemonic, op>, SMEPseudo2Instr<NAME, 1>; 2552 2553 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR8, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>; 2554 2555 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange2s4>; 2556} 2557 2558// SME2 multi-vec indexed long long MLA one source 64-bit 2559 2560class sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op> 2561 : I<(outs MatrixOp64:$ZAda), 2562 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR16:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 2563 mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", 2564 "", []>, Sched<[]> { 2565 bits<4> Zm; 2566 bits<2> Rv; 2567 bits<3> i; 2568 bits<5> Zn; 2569 bits<2> imm2; 2570 let Inst{31-20} = 0b110000011000; 2571 let Inst{19-16} = Zm; 2572 let Inst{15} = i{2}; 2573 let Inst{14-13} = Rv; 2574 let Inst{12} = 0b0; 2575 let Inst{11-10} = i{1-0}; 2576 let Inst{9-5} = Zn; 2577 let Inst{4-3} = op; 2578 let Inst{2} = 0b0; 2579 let Inst{1-0} = imm2; 2580 2581 let Constraints = "$ZAda = $_ZAda"; 2582} 2583 2584multiclass sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2585 def NAME : sme2_mla_ll_array_index_64b<mnemonic, op>, SMEPseudo2Instr<NAME, 1>; 2586 2587 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm2s4range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 2588 2589 def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME, intrinsic, uimm2s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s4>; 2590} 2591 2592class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op, 2593 RegisterOperand vector_ty, 2594 string mnemonic> 2595 : I<(outs MatrixOp32:$ZAda), 2596 (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2597 vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 2598 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2599 "", []>, Sched<[]> { 2600 bits<4> Zm; 2601 bits<2> Rv; 2602 bits<4> i; 2603 bit imm; 2604 let Inst{31-20} = 0b110000010001; 2605 let Inst{19-16} = Zm; 2606 let Inst{15} = vg4; 2607 let Inst{14-13} = Rv; 2608 let Inst{12} = 0b0; 2609 let Inst{11-10} = i{3-2}; 2610 let Inst{5-3} = op; 2611 let Inst{2-1} = i{1-0}; 2612 let Inst{0} = imm; 2613 2614 let Constraints = "$ZAda = $_ZAda"; 2615} 2616 2617//SME2 multi-vec indexed long long MLA two sources 32-bit 2618 2619multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2620 def NAME: sme2_mla_ll_array_vg24_index_32b<0b0, op, ZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2621 bits<4> Zn; 2622 let Inst{9-6} = Zn; 2623 } 2624 2625 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>; 2626 2627 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange1s4>; 2628 2629 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2630 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>; 2631} 2632 2633// SME2 multi-vec indexed long long MLA four sources 32-bit 2634 2635multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2636 def NAME: sme2_mla_ll_array_vg24_index_32b<0b1, op, ZZZZ_b_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2637 bits<3> Zn; 2638 let Inst{9-7} = Zn; 2639 let Inst{6} = 0b0; 2640 } 2641 2642 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_b_mul_r, ZPR4b8, VectorIndexB32b_timm, SMEMatrixArray>; 2643 2644 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b8, nxv16i8, VectorIndexB32b_timm, tileslicerange1s4>; 2645 2646 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2647 (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB32b_timm:$i), 0>; 2648} 2649class sme2_mla_ll_array_vg24_index_64b<bit vg4, bits<2> op, 2650 RegisterOperand vector_ty, 2651 string mnemonic> 2652 : I<(outs MatrixOp64:$ZAda), 2653 (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2654 vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 2655 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", 2656 "", []>, Sched<[]> { 2657 bits<4> Zm; 2658 bits<2> Rv; 2659 bits<3> i; 2660 bit imm; 2661 let Inst{31-20} = 0b110000011001; 2662 let Inst{19-16} = Zm; 2663 let Inst{15} = vg4; 2664 let Inst{14-13} = Rv; 2665 let Inst{12-11} = 0b00; 2666 let Inst{10} = i{2}; 2667 let Inst{5} = 0b0; 2668 let Inst{4-3} = op; 2669 let Inst{2-1} = i{1-0}; 2670 let Inst{0} = imm; 2671 2672 let Constraints = "$ZAda = $_ZAda"; 2673} 2674 2675// SME2 multi-vec indexed long long MLA two sources 64-bit 2676 2677multiclass sme2_mla_ll_array_vg2_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2678 def NAME: sme2_mla_ll_array_vg24_index_64b<0b0, op, ZZ_h_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2679 bits<4> Zn; 2680 let Inst{9-6} = Zn; 2681 } 2682 2683 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 2684 2685 def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange1s4>; 2686 2687 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2688 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 0>; 2689} 2690 2691// SME2 multi-vec indexed long long MLA four sources 64-bit 2692 2693multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { 2694 def NAME: sme2_mla_ll_array_vg24_index_64b<0b1, op, ZZZZ_h_mul_r, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2695 bits<3> Zn; 2696 let Inst{9-7} = Zn; 2697 let Inst{6} = 0b0; 2698 } 2699 2700 def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, uimm1s4range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; 2701 2702 def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, uimm1s4range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange1s4>; 2703 2704 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", 2705 (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i), 0>; 2706} 2707 2708 2709//SME2 multiple and single vector long long FMA one source 2710 2711class sme2_mla_ll_array_single<string mnemonic, bits<4> op, 2712 MatrixOperand matrix_ty, ZPRRegOp vector_ty, 2713 ZPRRegOp zpr_ty> 2714 : I<(outs matrix_ty:$ZAda), 2715 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm, 2716 vector_ty:$Zn, zpr_ty:$Zm), 2717 mnemonic, "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2718 "", []>, Sched<[]> { 2719 bits<4> Zm; 2720 bits<2> Rv; 2721 bits<5> Zn; 2722 bits<2> imm; 2723 let Inst{31-23} = 0b110000010; 2724 let Inst{22} = op{3}; //sz 2725 let Inst{21-20} = 0b10; 2726 let Inst{19-16} = Zm; 2727 let Inst{15} = 0b0; 2728 let Inst{14-13} = Rv; 2729 let Inst{12-10} = 0b001; 2730 let Inst{9-5} = Zn; 2731 let Inst{4-2} = op{2-0}; 2732 let Inst{1-0} = imm; 2733 2734 let Constraints = "$ZAda = $_ZAda"; 2735} 2736 2737multiclass sme2_mla_ll_array_single<string mnemonic, bits<4> op, 2738 MatrixOperand matrix_ty, ZPRRegOp vector_ty, 2739 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { 2740 def NAME : sme2_mla_ll_array_single<mnemonic, op, matrix_ty, vector_ty, zpr_ty>, SMEPseudo2Instr<NAME, 1>; 2741 2742 def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm2s4range, vector_ty, zpr_ty, SMEMatrixArray>; 2743 2744 def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME, intrinsic, uimm2s4range, zpr_ty, vt, tileslicerange2s4>; 2745} 2746 2747class sme2_mla_ll_array_vg24_single<bits<5> op, MatrixOperand matrix_ty, 2748 RegisterOperand vector_ty, ZPRRegOp zpr_ty, 2749 string mnemonic> 2750 : I<(outs matrix_ty:$ZAda), 2751 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2752 vector_ty:$Zn, zpr_ty:$Zm), 2753 mnemonic, "\t$ZAda[$Rv, $imm, " # !if(op{3}, "vgx4", "vgx2") # "], $Zn, $Zm", 2754 "", []>, Sched<[]> { 2755 bits<4> Zm; 2756 bits<2> Rv; 2757 bits<5> Zn; 2758 bit imm; 2759 let Inst{31-23} = 0b110000010; 2760 let Inst{22} = op{4}; //sz 2761 let Inst{21} = 0b1; 2762 let Inst{20} = op{3}; //vg4 2763 let Inst{19-16} = Zm; 2764 let Inst{15} = 0b0; 2765 let Inst{14-13} = Rv; 2766 let Inst{12-10} = 0b000; 2767 let Inst{9-5} = Zn; 2768 let Inst{4-2} = op{2-0}; 2769 let Inst{1} = 0b0; 2770 let Inst{0} = imm; 2771 2772 let Constraints = "$ZAda = $_ZAda"; 2773} 2774 2775//SME2 single-multi long long MLA two and four sources 2776 2777multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<5> op, 2778 MatrixOperand matrix_ty, 2779 RegisterOperand multi_vector_ty, 2780 ZPRRegOp zpr_ty> { 2781 def NAME: sme2_mla_ll_array_vg24_single<op, matrix_ty, multi_vector_ty, 2782 zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 2783 2784 def NAME # _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, uimm1s4range, multi_vector_ty, zpr_ty, SMEMatrixArray>; 2785 2786 def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm], $Zn, $Zm", 2787 (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; 2788} 2789 2790multiclass sme2_mla_ll_array_vg2_single<string mnemonic, bits<5> op, 2791 MatrixOperand matrix_ty, 2792 RegisterOperand multi_vector_ty, 2793 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { 2794 2795 defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, op, matrix_ty, multi_vector_ty, zpr_ty>; 2796 2797 def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>; 2798} 2799 2800multiclass sme2_mla_ll_array_vg4_single<string mnemonic, bits<5> op, 2801 MatrixOperand matrix_ty, 2802 RegisterOperand multi_vector_ty, 2803 ZPRRegOp zpr_ty, ValueType vt, SDPatternOperator intrinsic> { 2804 defm NAME: sme2_mla_ll_array_vg24_single<mnemonic, op, matrix_ty, multi_vector_ty, zpr_ty>; 2805 2806 def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, uimm1s4range, zpr_ty, vt, tileslicerange1s4>; 2807} 2808 2809// SME2 multiple vectors long long MLA two sources 2810 2811class sme2_mla_ll_array_vg2_multi<bits<4> op, MatrixOperand matrix_ty, 2812 RegisterOperand vector_ty,string mnemonic> 2813 : I<(outs matrix_ty:$ZAda), 2814 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2815 vector_ty:$Zn, vector_ty:$Zm), 2816 mnemonic, "\t$ZAda[$Rv, $imm, vgx2], $Zn, $Zm", 2817 "", []>, Sched<[]> { 2818 bits<4> Zm; 2819 bits<2> Rv; 2820 bits<4> Zn; 2821 bit imm; 2822 let Inst{31-23} = 0b110000011; 2823 let Inst{22} = op{3}; // sz 2824 let Inst{21} = 0b1; 2825 let Inst{20-17} = Zm; 2826 let Inst{16-15} = 0b00; 2827 let Inst{14-13} = Rv; 2828 let Inst{12-10} = 0b000; 2829 let Inst{9-6} = Zn; 2830 let Inst{5} = 0b0; 2831 let Inst{4-2} = op{2-0}; 2832 let Inst{1} = 0b0; 2833 let Inst{0} = imm; 2834 2835 let Constraints = "$ZAda = $_ZAda"; 2836} 2837 2838multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<4> op, 2839 MatrixOperand matrix_ty, 2840 RegisterOperand vector_ty, 2841 ValueType vt, SDPatternOperator intrinsic> { 2842 def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 2843 2844 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range, vector_ty, SMEMatrixArray>; 2845 2846 def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, uimm1s4range, vt, tileslicerange1s4>; 2847 2848 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2849 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; 2850} 2851 2852// SME2 multiple vectors long long MLA four sources 2853 2854class sme2_mla_ll_array_vg4_multi<bits<4> op,MatrixOperand matrix_ty, 2855 RegisterOperand vector_ty, 2856 string mnemonic> 2857 : I<(outs matrix_ty:$ZAda), 2858 (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, 2859 vector_ty:$Zn, vector_ty:$Zm), 2860 mnemonic, "\t$ZAda[$Rv, $imm, vgx4], $Zn, $Zm", 2861 "", []>, Sched<[]> { 2862 bits<3> Zm; 2863 bits<2> Rv; 2864 bits<3> Zn; 2865 bit imm; 2866 let Inst{31-23} = 0b110000011; 2867 let Inst{22} = op{3}; // sz 2868 let Inst{21} = 0b1; 2869 let Inst{20-18} = Zm; 2870 let Inst{17-15} = 0b010; 2871 let Inst{14-13} = Rv; 2872 let Inst{12-10} = 0b000; 2873 let Inst{9-7} = Zn; 2874 let Inst{6-5} = 0b00; 2875 let Inst{4-2} = op{2-0}; 2876 let Inst{1} = 0b0; 2877 let Inst{0} = imm; 2878 2879 let Constraints = "$ZAda = $_ZAda"; 2880} 2881 2882multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<4> op, 2883 MatrixOperand matrix_ty, 2884 RegisterOperand vector_ty, 2885 ValueType vt, SDPatternOperator intrinsic> { 2886 def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; 2887 2888 def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, uimm1s4range, vector_ty, SMEMatrixArray>; 2889 2890 def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, uimm1s4range, vt, tileslicerange1s4>; 2891 2892 def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", 2893 (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; 2894} 2895 2896//===----------------------------------------------------------------------===// 2897// SME2 Outer Product and Accumulate 2898 2899multiclass sme2_int_mopx_tile<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2900 def NAME : sme_int_outer_product_inst<op, 0b0, 0b1, TileOp32, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> { 2901 bits<2> ZAda; 2902 let Inst{1-0} = ZAda; 2903 let Inst{2} = 0b0; 2904 } 2905 2906 def _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 2907 2908 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv8i1, nxv8i16>; 2909} 2910 2911multiclass sme2_int_bmopx_tile<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 2912 def NAME : sme_outer_product_widening_inst<op, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1>; 2913 2914 def _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; 2915 2916 def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, intrinsic, timm32_0_3, nxv4i1, nxv4i32>; 2917} 2918 2919//===----------------------------------------------------------------------===/// 2920// SME2 Zero Lookup Table. 2921class sme2_zero_zt<string mnemonic, bits<4> opc> 2922 : I<(outs ZTR:$ZT), (ins ), 2923 mnemonic, "\t\\{ $ZT \\}", 2924 "", []>, Sched<[]> { 2925 let Inst{31-4} = 0b1100000001001000000000000000; 2926 let Inst{3-0} = opc; 2927} 2928 2929//===----------------------------------------------------------------------===// 2930// SME2 lookup table load/store 2931class sme2_spill_fill_vector<string mnemonic, bits<8> opc> 2932 : I<!if(opc{7}, (outs ), (outs ZTR:$ZTt)), 2933 !if(opc{7}, (ins ZTR:$ZTt, GPR64sp:$Rn), (ins GPR64sp:$Rn)), 2934 mnemonic, "\t$ZTt, [$Rn]", 2935 "", []>, Sched<[]> { 2936 bits<5> Rn; 2937 let Inst{31-22} = 0b1110000100; 2938 let Inst{21-16} = opc{7-2}; 2939 let Inst{15-10} = 0b100000; 2940 let Inst{9-5} = Rn; 2941 let Inst{4-2} = 0b000; 2942 let Inst{1-0} = opc{1-0}; 2943 2944 let mayLoad = !not(opc{7}); 2945 let mayStore = opc{7}; 2946} 2947 2948//===----------------------------------------------------------------------===/// 2949// SME2 move to/from lookup table 2950class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc> 2951 : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3), 2952 mnemonic, "\t$Rt, $ZTt$imm3", 2953 "", []>, Sched<[]> { 2954 bits<3> imm3; 2955 bits<5> Rt; 2956 let Inst{31-15} = 0b11000000010011000; 2957 let Inst{14-12} = imm3; 2958 let Inst{11-5} = opc; 2959 let Inst{4-0} = Rt; 2960} 2961 2962class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc> 2963 : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt), 2964 mnemonic, "\t$ZTt$imm3, $Rt", 2965 "", []>, Sched<[]> { 2966 bits<3> imm3; 2967 bits<5> Rt; 2968 let Inst{31-15} = 0b11000000010011100; 2969 let Inst{14-12} = imm3; 2970 let Inst{11-5} = opc; 2971 let Inst{4-0} = Rt; 2972} 2973 2974//===----------------------------------------------------------------------===// 2975// SME2 lookup table expand one register 2976class sme2_luti_vector_index<bits<2> sz, bits<7> opc, RegisterOperand vector_ty, 2977 AsmVectorIndexOpnd index_ty, string mnemonic> 2978 : I<(outs vector_ty:$Zd), 2979 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 2980 mnemonic, "\t$Zd, $ZTt, $Zn$i", 2981 "", []>, Sched<[]> { 2982 bits<5> Zn; 2983 bits<5> Zd; 2984 let Inst{31-19} = 0b1100000011001; 2985 let Inst{18-14} = opc{6-2}; 2986 let Inst{13-12} = sz; 2987 let Inst{11-10} = opc{1-0}; 2988 let Inst{9-5} = Zn; 2989 let Inst{4-0} = Zd; 2990} 2991 2992class sme2_luti2_vector_index<bits<2> sz, RegisterOperand vector_ty, 2993 string mnemonic> 2994 : sme2_luti_vector_index<sz, {1,?,?,?,?,0,0}, vector_ty, VectorIndexB, mnemonic> { 2995 bits<4> i; 2996 let Inst{17-14} = i; 2997} 2998 2999multiclass sme2_luti2_vector_index<string mnemonic> { 3000 def _B : sme2_luti2_vector_index<0b00, ZPR8, mnemonic>; 3001 def _H : sme2_luti2_vector_index<0b01, ZPR16, mnemonic>; 3002 def _S : sme2_luti2_vector_index<0b10, ZPR32, mnemonic>; 3003} 3004 3005class sme2_luti4_vector_index<bits<2> sz, RegisterOperand vector_ty, 3006 string mnemonic> 3007 : sme2_luti_vector_index<sz, {0,1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { 3008 bits<3> i; 3009 let Inst{16-14} = i; 3010} 3011 3012multiclass sme2_luti4_vector_index<string mnemonic> { 3013 def _B : sme2_luti4_vector_index<0b00, ZPR8, mnemonic>; 3014 def _H : sme2_luti4_vector_index<0b01, ZPR16, mnemonic>; 3015 def _S : sme2_luti4_vector_index<0b10, ZPR32, mnemonic>; 3016} 3017 3018// SME2 lookup table expand two contiguous registers 3019class sme2_luti_vector_vg2_index<bits<2> sz, bits<6> opc, RegisterOperand vector_ty, 3020 AsmVectorIndexOpnd index_ty, string mnemonic> 3021 : I<(outs vector_ty:$Zd), 3022 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 3023 mnemonic, "\t$Zd, $ZTt, $Zn$i", 3024 "", []>, Sched<[]> { 3025 bits<5> Zn; 3026 bits<4> Zd; 3027 let Inst{31-19} = 0b1100000010001; 3028 let Inst{18-15} = opc{5-2}; 3029 let Inst{14} = 0b1; 3030 let Inst{13-12} = sz; 3031 let Inst{11-10} = opc{1-0}; 3032 let Inst{9-5} = Zn; 3033 let Inst{4-1} = Zd; 3034 let Inst{0} = 0b0; 3035} 3036 3037class sme2_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 3038 string mnemonic> 3039 : sme2_luti_vector_vg2_index<sz, {1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { 3040 bits<3> i; 3041 let Inst{17-15} = i; 3042} 3043 3044multiclass sme2_luti2_vector_vg2_index<string mnemonic> { 3045 def _B : sme2_luti2_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; 3046 def _H : sme2_luti2_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; 3047 def _S : sme2_luti2_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; 3048} 3049 3050class sme2_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 3051 string mnemonic> 3052 : sme2_luti_vector_vg2_index<sz, {0,1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { 3053 bits<2> i; 3054 let Inst{16-15} = i; 3055} 3056 3057multiclass sme2_luti4_vector_vg2_index<string mnemonic> { 3058 def _B : sme2_luti4_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; 3059 def _H : sme2_luti4_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; 3060 def _S : sme2_luti4_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; 3061} 3062 3063// SME2 lookup table expand four contiguous registers 3064class sme2_luti_vector_vg4_index<bits<2> sz, bits<5>opc, RegisterOperand vector_ty, 3065 AsmVectorIndexOpnd index_ty, string mnemonic> 3066 : I<(outs vector_ty:$Zd), 3067 (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 3068 mnemonic, "\t$Zd, $ZTt, $Zn$i", 3069 "", []>, Sched<[]> { 3070 bits<5> Zn; 3071 bits<3> Zd; 3072 let Inst{31-19} = 0b1100000010001; 3073 let Inst{18-16} = opc{4-2}; 3074 let Inst{15-14} = 0b10; 3075 let Inst{13-12} = sz; 3076 let Inst{11-10} = opc{1-0}; 3077 let Inst{9-5} = Zn; 3078 let Inst{4-2} = Zd; 3079 let Inst{1-0} = 0b00; 3080} 3081 3082class sme2_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 3083 string mnemonic> 3084 : sme2_luti_vector_vg4_index<sz, {1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { 3085 bits<2> i; 3086 let Inst{17-16} = i; 3087} 3088 3089multiclass sme2_luti2_vector_vg4_index<string mnemonic> { 3090 def _B : sme2_luti2_vector_vg4_index<0b00, ZZZZ_b_mul_r, mnemonic>; 3091 def _H : sme2_luti2_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; 3092 def _S : sme2_luti2_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; 3093} 3094 3095class sme2_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 3096 string mnemonic> 3097 : sme2_luti_vector_vg4_index<sz, {0,1,?,0,0}, vector_ty, VectorIndexD, mnemonic> { 3098 bits<1> i; 3099 let Inst{16} = i; 3100} 3101 3102multiclass sme2_luti4_vector_vg4_index<string mnemonic> { 3103 def _H : sme2_luti4_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; 3104 def _S : sme2_luti4_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; 3105} 3106 3107//===----------------------------------------------------------------------===// 3108// SME2 MOV 3109class sme2_mova_vec_to_tile_vg2_multi_base<bits<2> sz, bit v, 3110 RegisterOperand tile_ty, 3111 Operand index_ty, 3112 RegisterOperand vector_ty, 3113 string mnemonic> 3114 : I<(outs tile_ty:$ZAd), 3115 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, vector_ty:$Zn), 3116 mnemonic, "\t$ZAd[$Rs, $imm], $Zn", 3117 "", []>, Sched<[]> { 3118 bits<2> Rs; 3119 bits<4> Zn; 3120 let Inst{31-24} = 0b11000000; 3121 let Inst{23-22} = sz; 3122 let Inst{21-16} = 0b000100; 3123 let Inst{15} = v; 3124 let Inst{14-13} = Rs; 3125 let Inst{12-10} = 0b000; 3126 let Inst{9-6} = Zn; 3127 let Inst{5-3} = 0b000; 3128 3129 let Constraints = "$ZAd = $_ZAd"; 3130} 3131 3132multiclass sme2_mova_vec_to_tile_or_array_aliases<int prefer, Instruction inst, 3133 RegisterOperand tile_or_array_ty, 3134 RegisterOperand rv_ty, 3135 Operand index_ty, 3136 RegisterOperand vector_ty, 3137 string mnemonic, 3138 string vg_acronym=""> { 3139 def : InstAlias<mnemonic # "\t$ZAd[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn", 3140 (inst tile_or_array_ty:$ZAd, rv_ty:$Rs, index_ty:$imm, vector_ty:$Zn), prefer>; 3141 3142} 3143 3144// SME2 move vector to tile, two registers 3145multiclass sme2_mova_vec_to_tile_vg2_multi_base<bit v, string mnemonic, SDPatternOperator intrinsic> { 3146 3147 def _B : sme2_mova_vec_to_tile_vg2_multi_base<0b00, v, 3148 !if(v, TileVectorOpV8, 3149 TileVectorOpH8), 3150 uimm3s2range, ZZ_b_mul_r, 3151 mnemonic>, SMEPseudo2Instr<NAME # _B, 1> { 3152 bits<3> imm; 3153 let Inst{2-0} = imm; 3154 } 3155 3156 def _H : sme2_mova_vec_to_tile_vg2_multi_base<0b01, v, 3157 !if(v, TileVectorOpV16, 3158 TileVectorOpH16), 3159 uimm2s2range, ZZ_h_mul_r, 3160 mnemonic>, SMEPseudo2Instr<NAME # _H, 1> { 3161 bits<1> ZAd; 3162 bits<2> imm; 3163 let Inst{2} = ZAd; 3164 let Inst{1-0} = imm; 3165 } 3166 3167 def _S : sme2_mova_vec_to_tile_vg2_multi_base<0b10, v, 3168 !if(v, TileVectorOpV32, 3169 TileVectorOpH32), 3170 uimm1s2range, ZZ_s_mul_r, 3171 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { 3172 bits<2> ZAd; 3173 bits<1> imm; 3174 let Inst{2-1} = ZAd; 3175 let Inst{0} = imm; 3176 } 3177 3178 def _D : sme2_mova_vec_to_tile_vg2_multi_base<0b11, v, 3179 !if(v, TileVectorOpV64, 3180 TileVectorOpH64), 3181 uimm0s2range, ZZ_d_mul_r, 3182 mnemonic>, SMEPseudo2Instr<NAME # _D, 1> { 3183 bits<3> ZAd; 3184 let Inst{2-0} = ZAd; 3185 } 3186 3187 def NAME # _B_PSEUDO : sme2_move_to_tile_pseudo<NAME # _B, sme_elm_idx0_0, uimm3s2range, ZZ_b_mul_r, SMEMatrixTileB>; 3188 def NAME # _H_PSEUDO : sme2_move_to_tile_pseudo<NAME # _H, sme_elm_idx0_1, uimm2s2range, ZZ_h_mul_r, SMEMatrixTileH>; 3189 def NAME # _S_PSEUDO : sme2_move_to_tile_pseudo<NAME # _S, sme_elm_idx0_3, uimm1s2range, ZZ_s_mul_r, SMEMatrixTileS>; 3190 def NAME # _D_PSEUDO : sme2_move_to_tile_pseudo<NAME # _D, sme_elm_idx0_7, uimm0s2range, ZZ_d_mul_r, SMEMatrixTileD>; 3191 3192 def : SME2_Tile_VG2_Multi_Pat<NAME # _B, intrinsic, sme_elm_idx0_0, nxv16i8, uimm3s2range, tileslicerange3s2>; 3193 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8i16, uimm2s2range, tileslicerange2s2>; 3194 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8f16, uimm2s2range, tileslicerange2s2>; 3195 def : SME2_Tile_VG2_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8bf16, uimm2s2range, tileslicerange2s2>; 3196 def : SME2_Tile_VG2_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4i32, uimm1s2range, tileslicerange1s2>; 3197 def : SME2_Tile_VG2_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4f32, uimm1s2range, tileslicerange1s2>; 3198 def : SME2_Tile_VG2_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2i64, uimm0s2range, tileslicerange0s2>; 3199 def : SME2_Tile_VG2_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2f64, uimm0s2range, tileslicerange0s2>; 3200 3201 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), 3202 !if(v, TileVectorOpV8, 3203 TileVectorOpH8), 3204 MatrixIndexGPR32Op12_15, 3205 uimm3s2range, ZZ_b_mul_r, 3206 "mov">; 3207 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), 3208 !if(v, TileVectorOpV16, 3209 TileVectorOpH16), 3210 MatrixIndexGPR32Op12_15, 3211 uimm2s2range, ZZ_h_mul_r, 3212 "mov">; 3213 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), 3214 !if(v, TileVectorOpV32, 3215 TileVectorOpH32), 3216 MatrixIndexGPR32Op12_15, 3217 uimm1s2range, ZZ_s_mul_r, 3218 "mov">; 3219 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), 3220 !if(v, TileVectorOpV64, 3221 TileVectorOpH64), 3222 MatrixIndexGPR32Op12_15, 3223 uimm0s2range, ZZ_d_mul_r, 3224 "mov">; 3225 3226 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3227 !if(v, TileVectorOpV8, 3228 TileVectorOpH8), 3229 MatrixIndexGPR32Op12_15, 3230 uimm3s2range, ZZ_b_mul_r, 3231 "mova">; 3232 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3233 !if(v, TileVectorOpV16, 3234 TileVectorOpH16), 3235 MatrixIndexGPR32Op12_15, 3236 uimm2s2range, ZZ_h_mul_r, 3237 "mova">; 3238 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3239 !if(v, TileVectorOpV32, 3240 TileVectorOpH32), 3241 MatrixIndexGPR32Op12_15, 3242 uimm1s2range, ZZ_s_mul_r, 3243 "mova">; 3244 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3245 !if(v, TileVectorOpV64, 3246 TileVectorOpH64), 3247 MatrixIndexGPR32Op12_15, 3248 uimm0s2range, ZZ_d_mul_r, 3249 "mova">; 3250 3251 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3252 !if(v, TileVectorOpV8, 3253 TileVectorOpH8), 3254 MatrixIndexGPR32Op12_15, 3255 uimm3s2range, ZZ_b_mul_r, 3256 "mova">; 3257 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3258 !if(v, TileVectorOpV16, 3259 TileVectorOpH16), 3260 MatrixIndexGPR32Op12_15, 3261 uimm2s2range, ZZ_h_mul_r, 3262 "mova">; 3263 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3264 !if(v, TileVectorOpV32, 3265 TileVectorOpH32), 3266 MatrixIndexGPR32Op12_15, 3267 uimm1s2range, ZZ_s_mul_r, 3268 "mova">; 3269 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3270 !if(v, TileVectorOpV64, 3271 TileVectorOpH64), 3272 MatrixIndexGPR32Op12_15, 3273 uimm0s2range, ZZ_d_mul_r, 3274 "mova">; 3275} 3276 3277multiclass sme2_mova_vec_to_tile_vg2_multi<string mnemonic, 3278 SDPatternOperator int_h, SDPatternOperator int_v>{ 3279 defm _H : sme2_mova_vec_to_tile_vg2_multi_base<0b0, mnemonic, int_h>; 3280 defm _V : sme2_mova_vec_to_tile_vg2_multi_base<0b1, mnemonic, int_v>; 3281} 3282 3283class sme2_mova_vec_to_tile_vg4_multi_base<bits<2> sz, bit v, bits<3> op, 3284 RegisterOperand tile_ty, 3285 Operand index_ty, 3286 RegisterOperand vector_ty, 3287 string mnemonic> 3288 : I<(outs tile_ty:$ZAd), 3289 (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, 3290 vector_ty:$Zn), 3291 mnemonic, 3292 "\t$ZAd[$Rs, $imm], $Zn", 3293 "", []>, Sched<[]> { 3294 bits<2> Rs; 3295 bits<3> Zn; 3296 let Inst{31-24} = 0b11000000; 3297 let Inst{23-22} = sz; 3298 let Inst{21-16} = 0b000100; 3299 let Inst{15} = v; 3300 let Inst{14-13} = Rs; 3301 let Inst{12-10} = 0b001; 3302 let Inst{9-7} = Zn; 3303 let Inst{6-3} = 0b0000; 3304 let Inst{2-0} = op; 3305 let Constraints = "$ZAd = $_ZAd"; 3306} 3307 3308// SME2 move vector to tile, four registers 3309multiclass sme2_mova_vec_to_tile_vg4_multi_base<bit v, string mnemonic, SDPatternOperator intrinsic> { 3310 3311 def _B : sme2_mova_vec_to_tile_vg4_multi_base<0b00, v, {0,?,?}, 3312 !if(v, TileVectorOpV8, 3313 TileVectorOpH8), 3314 uimm2s4range, ZZZZ_b_mul_r, 3315 mnemonic>, SMEPseudo2Instr<NAME # _B, 1> { 3316 bits<2> imm; 3317 let Inst{1-0} = imm; 3318 } 3319 3320 def _H : sme2_mova_vec_to_tile_vg4_multi_base<0b01, v, {0,?,?}, 3321 !if(v, TileVectorOpV16, 3322 TileVectorOpH16), 3323 uimm1s4range, ZZZZ_h_mul_r, 3324 mnemonic>, SMEPseudo2Instr<NAME # _H, 1> { 3325 bits<1> ZAd; 3326 bits<1> imm; 3327 let Inst{1} = ZAd; 3328 let Inst{0} = imm; 3329 } 3330 3331 def _S : sme2_mova_vec_to_tile_vg4_multi_base<0b10, v, {0,?,?}, 3332 !if(v, TileVectorOpV32, 3333 TileVectorOpH32), 3334 uimm0s4range, ZZZZ_s_mul_r, 3335 mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { 3336 bits<2> ZAd; 3337 let Inst{1-0} = ZAd; 3338 } 3339 3340 def _D : sme2_mova_vec_to_tile_vg4_multi_base<0b11, v, {?,?,?}, 3341 !if(v, TileVectorOpV64, 3342 TileVectorOpH64), 3343 uimm0s4range, ZZZZ_d_mul_r, 3344 mnemonic>, SMEPseudo2Instr<NAME # _D, 1> { 3345 bits<3> ZAd; 3346 let Inst{2-0} = ZAd; 3347 } 3348 3349 def NAME # _B_PSEUDO : sme2_move_to_tile_pseudo<NAME # _B, sme_elm_idx0_0, uimm2s4range, ZZZZ_b_mul_r, SMEMatrixTileB>; 3350 def NAME # _H_PSEUDO : sme2_move_to_tile_pseudo<NAME # _H, sme_elm_idx0_1, uimm1s4range, ZZZZ_h_mul_r, SMEMatrixTileH>; 3351 def NAME # _S_PSEUDO : sme2_move_to_tile_pseudo<NAME # _S, sme_elm_idx0_3, uimm0s4range, ZZZZ_s_mul_r, SMEMatrixTileS>; 3352 def NAME # _D_PSEUDO : sme2_move_to_tile_pseudo<NAME # _D, sme_elm_idx0_7, uimm0s4range, ZZZZ_d_mul_r, SMEMatrixTileD>; 3353 3354 def : SME2_Tile_VG4_Multi_Pat<NAME # _B, intrinsic, sme_elm_idx0_0, nxv16i8, uimm2s4range, tileslicerange2s4>; 3355 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8i16, uimm1s4range, tileslicerange1s4>; 3356 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8f16, uimm1s4range, tileslicerange1s4>; 3357 def : SME2_Tile_VG4_Multi_Pat<NAME # _H, intrinsic, sme_elm_idx0_1, nxv8bf16, uimm1s4range, tileslicerange1s4>; 3358 def : SME2_Tile_VG4_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4i32, uimm0s4range, tileslicerange0s4>; 3359 def : SME2_Tile_VG4_Multi_Pat<NAME # _S, intrinsic, sme_elm_idx0_3, nxv4f32, uimm0s4range, tileslicerange0s4>; 3360 def : SME2_Tile_VG4_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2i64, uimm0s4range, tileslicerange0s4>; 3361 def : SME2_Tile_VG4_Multi_Pat<NAME # _D, intrinsic, sme_elm_idx0_7, nxv2f64, uimm0s4range, tileslicerange0s4>; 3362 3363 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), 3364 !if(v, TileVectorOpV8, 3365 TileVectorOpH8), 3366 MatrixIndexGPR32Op12_15, 3367 uimm2s4range, ZZZZ_b_mul_r, 3368 "mov">; 3369 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), 3370 !if(v, TileVectorOpV16, 3371 TileVectorOpH16), 3372 MatrixIndexGPR32Op12_15, 3373 uimm1s4range, ZZZZ_h_mul_r, 3374 "mov">; 3375 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), 3376 !if(v, TileVectorOpV32, 3377 TileVectorOpH32), 3378 MatrixIndexGPR32Op12_15, 3379 uimm0s4range, ZZZZ_s_mul_r, 3380 "mov">; 3381 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), 3382 !if(v, TileVectorOpV64, 3383 TileVectorOpH64), 3384 MatrixIndexGPR32Op12_15, 3385 uimm0s4range, ZZZZ_d_mul_r, 3386 "mov">; 3387 3388 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), 3389 !if(v, TileVectorOpV8, 3390 TileVectorOpH8), 3391 MatrixIndexGPR32Op12_15, 3392 uimm2s4range, ZZZZ_b_mul_r, 3393 "mova">; 3394 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), 3395 !if(v, TileVectorOpV16, 3396 TileVectorOpH16), 3397 MatrixIndexGPR32Op12_15, 3398 uimm1s4range, ZZZZ_h_mul_r, 3399 "mova">; 3400 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), 3401 !if(v, TileVectorOpV32, 3402 TileVectorOpH32), 3403 MatrixIndexGPR32Op12_15, 3404 uimm0s4range, ZZZZ_s_mul_r, 3405 "mova">; 3406 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), 3407 !if(v, TileVectorOpV64, 3408 TileVectorOpH64), 3409 MatrixIndexGPR32Op12_15, 3410 uimm0s4range, ZZZZ_d_mul_r, 3411 "mova">; 3412 3413} 3414 3415multiclass sme2_mova_vec_to_tile_vg4_multi<string mnemonic, 3416 SDPatternOperator int_h, SDPatternOperator int_v>{ 3417 defm _H : sme2_mova_vec_to_tile_vg4_multi_base<0b0, mnemonic, int_h>; 3418 defm _V : sme2_mova_vec_to_tile_vg4_multi_base<0b1, mnemonic, int_v>; 3419} 3420 3421// SME Move into Array 3422class sme2_mova_vec_to_array_vg24_multi< bits<5> op, RegisterOperand array_ty, 3423 RegisterOperand vector_ty, 3424 string mnemonic, 3425 string vg_acronym=""> 3426 : I<(outs array_ty:$ZAd), 3427 (ins array_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm, 3428 vector_ty:$Zn), 3429 mnemonic, "\t$ZAd[$Rs, $imm, " # vg_acronym # "], $Zn", 3430 "", []>, Sched<[]> { 3431 bits<2> Rs; 3432 bits<3> imm; 3433 let Inst{31-15} = 0b11000000000001000; 3434 let Inst{14-13} = Rs; 3435 let Inst{12-11} = 0b01; 3436 let Inst{10-6} = op; 3437 let Inst{5-3} = 0b000; 3438 let Inst{2-0} = imm; 3439 3440 let Constraints = "$ZAd = $_ZAd"; 3441} 3442 3443// MOVA (vector to array, two registers) 3444multiclass sme2_mova_vec_to_array_vg2_multi<string mnemonic, SDPatternOperator intrinsic> { 3445 def NAME : sme2_mova_vec_to_array_vg24_multi<{0,?,?,?,?}, MatrixOp64, 3446 ZZ_d_mul_r, mnemonic, "vgx2">, SMEPseudo2Instr<NAME, 1> { 3447 bits<4> Zn; 3448 let Inst{9-6} = Zn; 3449 } 3450 3451 def NAME # _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, ZZ_d_mul_r, SMEMatrixArray>; 3452 3453 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv2i64, sme_elm_idx0_7, tileslice16>; 3454 def : SME2_ZA_VG1x2_Multi_Pat<NAME, intrinsic, nxv2f64, sme_elm_idx0_7, tileslice16>; 3455 3456 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3457 MatrixOp8, 3458 MatrixIndexGPR32Op8_11, 3459 sme_elm_idx0_7, ZZ_b_mul_r, 3460 "mova">; 3461 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3462 MatrixOp16, 3463 MatrixIndexGPR32Op8_11, 3464 sme_elm_idx0_7, ZZ_h_mul_r, 3465 "mova">; 3466 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3467 MatrixOp32, 3468 MatrixIndexGPR32Op8_11, 3469 sme_elm_idx0_7, ZZ_s_mul_r, 3470 "mova">; 3471 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3472 MatrixOp64, 3473 MatrixIndexGPR32Op8_11, 3474 sme_elm_idx0_7, ZZ_d_mul_r, 3475 "mova">; 3476 3477 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3478 MatrixOp8, 3479 MatrixIndexGPR32Op8_11, 3480 sme_elm_idx0_7, ZZ_b_mul_r, 3481 "mova", "vgx2">; 3482 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3483 MatrixOp16, 3484 MatrixIndexGPR32Op8_11, 3485 sme_elm_idx0_7, ZZ_h_mul_r, 3486 "mova", "vgx2">; 3487 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3488 MatrixOp32, 3489 MatrixIndexGPR32Op8_11, 3490 sme_elm_idx0_7, ZZ_s_mul_r, 3491 "mova", "vgx2">; 3492 3493 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3494 MatrixOp8, 3495 MatrixIndexGPR32Op8_11, 3496 sme_elm_idx0_7, ZZ_b_mul_r, 3497 "mov">; 3498 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3499 MatrixOp16, 3500 MatrixIndexGPR32Op8_11, 3501 sme_elm_idx0_7, ZZ_h_mul_r, 3502 "mov">; 3503 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3504 MatrixOp32, 3505 MatrixIndexGPR32Op8_11, 3506 sme_elm_idx0_7, ZZ_s_mul_r, 3507 "mov">; 3508 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3509 MatrixOp64, 3510 MatrixIndexGPR32Op8_11, 3511 sme_elm_idx0_7, ZZ_d_mul_r, 3512 "mov">; 3513 3514 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3515 MatrixOp8, 3516 MatrixIndexGPR32Op8_11, 3517 sme_elm_idx0_7, ZZ_b_mul_r, 3518 "mov", "vgx2">; 3519 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3520 MatrixOp16, 3521 MatrixIndexGPR32Op8_11, 3522 sme_elm_idx0_7, ZZ_h_mul_r, 3523 "mov", "vgx2">; 3524 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3525 MatrixOp32, 3526 MatrixIndexGPR32Op8_11, 3527 sme_elm_idx0_7, ZZ_s_mul_r, 3528 "mov", "vgx2">; 3529 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), 3530 MatrixOp64, 3531 MatrixIndexGPR32Op8_11, 3532 sme_elm_idx0_7, ZZ_d_mul_r, 3533 "mov", "vgx2">; 3534} 3535 3536// MOVA (vector to array, four registers) 3537multiclass sme2_mova_vec_to_array_vg4_multi<string mnemonic, SDPatternOperator intrinsic> { 3538 def NAME : sme2_mova_vec_to_array_vg24_multi<{1,?,?,?,0}, MatrixOp64, 3539 ZZZZ_d_mul_r, mnemonic, "vgx4">, SMEPseudo2Instr<NAME, 1> { 3540 bits<3> Zn; 3541 let Inst{9-7} = Zn; 3542 } 3543 3544 def NAME # _PSEUDO : sme2_move_to_za_pseudo<NAME, sme_elm_idx0_7, ZZZZ_d_mul_r, SMEMatrixArray>; 3545 3546 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv2i64, sme_elm_idx0_7, tileslice16>; 3547 def : SME2_ZA_VG1x4_Multi_Pat<NAME, intrinsic, nxv2f64, sme_elm_idx0_7, tileslice16>; 3548 3549 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3550 MatrixOp8, 3551 MatrixIndexGPR32Op8_11, 3552 sme_elm_idx0_7, ZZZZ_b_mul_r, 3553 "mova">; 3554 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3555 MatrixOp16, 3556 MatrixIndexGPR32Op8_11, 3557 sme_elm_idx0_7, ZZZZ_h_mul_r, 3558 "mova">; 3559 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3560 MatrixOp32, 3561 MatrixIndexGPR32Op8_11, 3562 sme_elm_idx0_7, ZZZZ_s_mul_r, 3563 "mova">; 3564 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3565 MatrixOp64, 3566 MatrixIndexGPR32Op8_11, 3567 sme_elm_idx0_7, ZZZZ_d_mul_r, 3568 "mova">; 3569 3570 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3571 MatrixOp8, 3572 MatrixIndexGPR32Op8_11, 3573 sme_elm_idx0_7, ZZZZ_b_mul_r, 3574 "mova", "vgx4">; 3575 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3576 MatrixOp16, 3577 MatrixIndexGPR32Op8_11, 3578 sme_elm_idx0_7, ZZZZ_h_mul_r, 3579 "mova", "vgx4">; 3580 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3581 MatrixOp32, 3582 MatrixIndexGPR32Op8_11, 3583 sme_elm_idx0_7, ZZZZ_s_mul_r, 3584 "mova", "vgx4">; 3585 3586 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3587 MatrixOp8, 3588 MatrixIndexGPR32Op8_11, 3589 sme_elm_idx0_7, ZZZZ_b_mul_r, 3590 "mov">; 3591 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3592 MatrixOp16, 3593 MatrixIndexGPR32Op8_11, 3594 sme_elm_idx0_7, ZZZZ_h_mul_r, 3595 "mov">; 3596 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3597 MatrixOp32, 3598 MatrixIndexGPR32Op8_11, 3599 sme_elm_idx0_7, ZZZZ_s_mul_r, 3600 "mov">; 3601 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3602 MatrixOp64, 3603 MatrixIndexGPR32Op8_11, 3604 sme_elm_idx0_7, ZZZZ_d_mul_r, 3605 "mov">; 3606 3607 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3608 MatrixOp8, 3609 MatrixIndexGPR32Op8_11, 3610 sme_elm_idx0_7, ZZZZ_b_mul_r, 3611 "mov", "vgx4">; 3612 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3613 MatrixOp16, 3614 MatrixIndexGPR32Op8_11, 3615 sme_elm_idx0_7, ZZZZ_h_mul_r, 3616 "mov", "vgx4">; 3617 defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), 3618 MatrixOp32, 3619 MatrixIndexGPR32Op8_11, 3620 sme_elm_idx0_7, ZZZZ_s_mul_r, 3621 "mov", "vgx4">; 3622 defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), 3623 MatrixOp64, 3624 MatrixIndexGPR32Op8_11, 3625 sme_elm_idx0_7, ZZZZ_d_mul_r, 3626 "mov", "vgx4">; 3627 3628} 3629 3630class sme2_mova_tile_to_vec_vg2_multi_base<bits<2> sz, bit v, bits<3> op, 3631 RegisterOperand vector_ty, 3632 RegisterOperand tile_ty, 3633 Operand index_ty, 3634 string mnemonic> 3635 : I<!if(op{1}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), 3636 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 3637 mnemonic, 3638 "\t$Zd, $ZAn[$Rs, $imm]", 3639 "", []>, Sched<[]> { 3640 bits<4> Zd; 3641 bits<2> Rs; 3642 let Inst{31-24} = 0b11000000; 3643 let Inst{23-22} = sz; 3644 let Inst{21-16} = 0b000110; 3645 let Inst{15} = v; 3646 let Inst{14-13} = Rs; 3647 let Inst{12-11} = 0b00; 3648 let Inst{10-8} = op; 3649 let Inst{4-1} = Zd; 3650 let Inst{0} = 0b0; 3651 3652 let Constraints = !if(op{1}, "$ZAn = $_ZAn", ""); 3653} 3654 3655multiclass sme2_mova_tile_or_array_to_vec_aliases<int op, Instruction inst, 3656 RegisterOperand vector_ty, 3657 RegisterOperand tile_or_array_ty, 3658 RegisterOperand rv_ty, 3659 Operand index_ty, 3660 string mnemonic, 3661 string vg_acronym=""> { 3662def : InstAlias<mnemonic # "\t$Zd, $ZAn[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", 3663 (inst vector_ty:$Zd, tile_or_array_ty:$ZAn, rv_ty:$Rs, index_ty:$imm), op>; 3664 3665} 3666 3667multiclass sme2_mova_tile_to_vec_vg2_multi_inst<bit v, bits<3> opc, string mnemonic> { 3668 3669 def _B : sme2_mova_tile_to_vec_vg2_multi_base<0b00, v, opc, ZZ_b_mul_r, 3670 !if(v, TileVectorOpV8, 3671 TileVectorOpH8), 3672 uimm3s2range, mnemonic> { 3673 bits<3> imm; 3674 let Inst{7-5} = imm; 3675 } 3676 3677 def _H : sme2_mova_tile_to_vec_vg2_multi_base<0b01, v, opc, ZZ_h_mul_r, 3678 !if(v, TileVectorOpV16, 3679 TileVectorOpH16), 3680 uimm2s2range, mnemonic> { 3681 bits<1> ZAn; 3682 bits<2> imm; 3683 let Inst{7} = ZAn; 3684 let Inst{6-5} = imm; 3685 } 3686 3687 def _S : sme2_mova_tile_to_vec_vg2_multi_base<0b10, v, opc, ZZ_s_mul_r, 3688 !if(v, TileVectorOpV32, 3689 TileVectorOpH32), 3690 uimm1s2range, mnemonic> { 3691 bits<2> ZAn; 3692 bits<1> imm; 3693 let Inst{7-6} = ZAn; 3694 let Inst{5} = imm; 3695 } 3696 3697 def _D : sme2_mova_tile_to_vec_vg2_multi_base<0b11, v, opc, ZZ_d_mul_r, 3698 !if(v, TileVectorOpV64, 3699 TileVectorOpH64), 3700 uimm0s2range, mnemonic> { 3701 bits<3> ZAn; 3702 let Inst{7-5} = ZAn; 3703 } 3704 3705 if !eq(mnemonic, "mova") then { 3706 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _B), 3707 ZZ_b_mul_r, 3708 !if(v, TileVectorOpV8, 3709 TileVectorOpH8), 3710 MatrixIndexGPR32Op12_15, 3711 uimm3s2range, "mov">; 3712 defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _H), 3713 ZZ_h_mul_r, 3714 !if(v, TileVectorOpV16, 3715 TileVectorOpH16), 3716 MatrixIndexGPR32Op12_15, 3717 uimm2s2range, "mov">; 3718 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), 3719 ZZ_s_mul_r, 3720 !if(v, TileVectorOpV32, 3721 TileVectorOpH32), 3722 MatrixIndexGPR32Op12_15, 3723 uimm1s2range, "mov">; 3724 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), 3725 ZZ_d_mul_r, 3726 !if(v, TileVectorOpV64, 3727 TileVectorOpH64), 3728 MatrixIndexGPR32Op12_15, 3729 uimm0s2range, "mov">; 3730 } 3731 3732 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _B), 3733 ZZ_b_mul_r, 3734 !if(v, TileVectorOpV8, 3735 TileVectorOpH8), 3736 MatrixIndexGPR32Op12_15, 3737 uimm3s2range, mnemonic>; 3738 defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _H), 3739 ZZ_h_mul_r, 3740 !if(v, TileVectorOpV16, 3741 TileVectorOpH16), 3742 MatrixIndexGPR32Op12_15, 3743 uimm2s2range, mnemonic>; 3744 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), 3745 ZZ_s_mul_r, 3746 !if(v, TileVectorOpV32, 3747 TileVectorOpH32), 3748 MatrixIndexGPR32Op12_15, 3749 uimm1s2range, mnemonic>; 3750 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), 3751 ZZ_d_mul_r, 3752 !if(v, TileVectorOpV64, 3753 TileVectorOpH64), 3754 MatrixIndexGPR32Op12_15, 3755 uimm0s2range, mnemonic>; 3756 3757} 3758 3759// SME2 move tile to vector, two registers 3760multiclass sme2_mova_tile_to_vec_vg2_multi<string mnemonic>{ 3761 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b000, mnemonic>; 3762 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b000, mnemonic>; 3763} 3764 3765// SME2p1 move tile to vector and zero tile, two registers 3766multiclass sme2p1_movaz_tile_to_vec_vg2<string mnemonic>{ 3767 defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b010, mnemonic>; 3768 defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b010, mnemonic>; 3769} 3770 3771class sme2_mova_tile_to_vec_vg4_multi_base<bits<2> sz, bit v, bits<6> op, 3772 RegisterOperand vector_ty, 3773 RegisterOperand tile_ty, 3774 Operand index_ty, 3775 string mnemonic> 3776 : I<!if(op{4}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), 3777 (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 3778 mnemonic, 3779 "\t$Zd, $ZAn[$Rs, $imm]", 3780 "", []>, Sched<[]> { 3781 bits<3> Zd; 3782 bits<2> Rs; 3783 let Inst{31-24} = 0b11000000; 3784 let Inst{23-22} = sz; 3785 let Inst{21-16} = 0b000110; 3786 let Inst{15} = v; 3787 let Inst{14-13} = Rs; 3788 let Inst{12-11} = 0b00; 3789 let Inst{10-5} = op{5-0}; 3790 let Inst{4-2} = Zd; 3791 let Inst{1-0} = 0b00; 3792 3793 let Constraints = !if(op{4}, "$ZAn = $_ZAn", ""); 3794} 3795 3796multiclass sme2_mova_tile_to_vec_vg4_multi_base<bit v, bits<3> opc, string mnemonic> { 3797 3798 def _B : sme2_mova_tile_to_vec_vg4_multi_base<0b00, v, {opc,0,?,?}, 3799 ZZZZ_b_mul_r, 3800 !if(v, TileVectorOpV8, 3801 TileVectorOpH8), 3802 uimm2s4range, mnemonic> { 3803 bits<2> imm; 3804 let Inst{6-5} = imm; 3805 } 3806 3807 def _H : sme2_mova_tile_to_vec_vg4_multi_base<0b01, v, {opc,0,?,?}, 3808 ZZZZ_h_mul_r, 3809 !if(v, TileVectorOpV16, 3810 TileVectorOpH16), 3811 uimm1s4range, mnemonic> { 3812 bits<1> ZAn; 3813 bits<1> imm; 3814 let Inst{6} = ZAn; 3815 let Inst{5} = imm; 3816 } 3817 3818 def _S : sme2_mova_tile_to_vec_vg4_multi_base<0b10, v, {opc,0,?,?}, 3819 ZZZZ_s_mul_r, 3820 !if(v, TileVectorOpV32, 3821 TileVectorOpH32), 3822 uimm0s4range, mnemonic> { 3823 bits<2> ZAn; 3824 let Inst{6-5} = ZAn; 3825 } 3826 3827 def _D : sme2_mova_tile_to_vec_vg4_multi_base<0b11, v, {opc,?,?,?}, 3828 ZZZZ_d_mul_r, 3829 !if(v, TileVectorOpV64, 3830 TileVectorOpH64), 3831 uimm0s4range, mnemonic> { 3832 bits<3> ZAn; 3833 let Inst{7-5} = ZAn; 3834 } 3835 3836 if !eq(mnemonic, "mova") then { 3837 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _B), 3838 ZZZZ_b_mul_r, 3839 !if(v, TileVectorOpV8, 3840 TileVectorOpH8), 3841 MatrixIndexGPR32Op12_15, 3842 uimm2s4range, "mov">; 3843 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _H), 3844 ZZZZ_h_mul_r, 3845 !if(v, TileVectorOpV16, 3846 TileVectorOpH16), 3847 MatrixIndexGPR32Op12_15, 3848 uimm1s4range, "mov">; 3849 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), 3850 ZZZZ_s_mul_r, 3851 !if(v, TileVectorOpV32, 3852 TileVectorOpH32), 3853 MatrixIndexGPR32Op12_15, 3854 uimm0s4range, "mov">; 3855 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), 3856 ZZZZ_d_mul_r, 3857 !if(v, TileVectorOpV64, 3858 TileVectorOpH64), 3859 MatrixIndexGPR32Op12_15, 3860 uimm0s4range, "mov">; 3861 } 3862 3863 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _B), 3864 ZZZZ_b_mul_r, 3865 !if(v, TileVectorOpV8, 3866 TileVectorOpH8), 3867 MatrixIndexGPR32Op12_15, 3868 uimm2s4range, mnemonic>; 3869 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _H), 3870 ZZZZ_h_mul_r, 3871 !if(v, TileVectorOpV16, 3872 TileVectorOpH16), 3873 MatrixIndexGPR32Op12_15, 3874 uimm1s4range, mnemonic>; 3875 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), 3876 ZZZZ_s_mul_r, 3877 !if(v, TileVectorOpV32, 3878 TileVectorOpH32), 3879 MatrixIndexGPR32Op12_15, 3880 uimm0s4range, mnemonic>; 3881 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), 3882 ZZZZ_d_mul_r, 3883 !if(v, TileVectorOpV64, 3884 TileVectorOpH64), 3885 MatrixIndexGPR32Op12_15, 3886 uimm0s4range, mnemonic>; 3887 3888} 3889 3890// SME2 move tile to vector, four registers 3891multiclass sme2_mova_tile_to_vec_vg4_multi<string mnemonic>{ 3892 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b100, mnemonic>; 3893 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b100, mnemonic>; 3894} 3895 3896// SME2p1 move tile to vector and zero tile, four registers 3897multiclass sme2p1_movaz_tile_to_vec_vg4<string mnemonic>{ 3898 defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b110, mnemonic>; 3899 defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b110, mnemonic>; 3900} 3901 3902 3903class sme2_mova_array_to_vec_vg24_multi<bits<4>op, RegisterOperand vector_ty, 3904 RegisterOperand array_ty, 3905 string mnemonic, string vg_acronym> 3906 : I<!if(op{2}, (outs vector_ty:$Zd, array_ty:$_ZAn), (outs vector_ty:$Zd)), 3907 (ins array_ty:$ZAn, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm), 3908 mnemonic, 3909 "\t$Zd, $ZAn[$Rs, $imm, " # vg_acronym # "]", 3910 "", []>, Sched<[]> { 3911 bits<2> Rs; 3912 bits<3> imm; 3913 let Inst{31-15} = 0b11000000000001100; 3914 let Inst{14-13} = Rs; 3915 let Inst{12-11} = 0b01; 3916 let Inst{10-8} = op{3-1}; 3917 let Inst{7-5} = imm; 3918 let Inst{1} = op{0}; 3919 let Inst{0} = 0b0; 3920 let Constraints = !if(op{2}, "$ZAn = $_ZAn", ""); 3921} 3922 3923// move array to vector, two registers. 3924multiclass sme2_mova_array_to_vec_vg2_multi<bits<3> opc, string mnemonic> { 3925 def NAME : sme2_mova_array_to_vec_vg24_multi<{opc,?}, ZZ_d_mul_r, MatrixOp64, 3926 mnemonic, "vgx2"> { 3927 bits<4> Zd; 3928 let Inst{4-1} = Zd; 3929 } 3930 3931 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3932 ZZ_b_mul_r, MatrixOp8, 3933 MatrixIndexGPR32Op8_11, 3934 sme_elm_idx0_7, mnemonic>; 3935 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3936 ZZ_h_mul_r, MatrixOp16, 3937 MatrixIndexGPR32Op8_11, 3938 sme_elm_idx0_7, mnemonic>; 3939 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3940 ZZ_s_mul_r, MatrixOp32, 3941 MatrixIndexGPR32Op8_11, 3942 sme_elm_idx0_7, mnemonic>; 3943 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3944 ZZ_d_mul_r, MatrixOp64, 3945 MatrixIndexGPR32Op8_11, 3946 sme_elm_idx0_7, mnemonic>; 3947 3948 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3949 ZZ_b_mul_r, MatrixOp8, 3950 MatrixIndexGPR32Op8_11, 3951 sme_elm_idx0_7, mnemonic, "vgx2">; 3952 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3953 ZZ_h_mul_r, MatrixOp16, 3954 MatrixIndexGPR32Op8_11, 3955 sme_elm_idx0_7, mnemonic, "vgx2">; 3956 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3957 ZZ_s_mul_r, MatrixOp32, 3958 MatrixIndexGPR32Op8_11, 3959 sme_elm_idx0_7, mnemonic, "vgx2">; 3960 3961 if !eq(mnemonic, "mova") then { 3962 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3963 ZZ_b_mul_r, MatrixOp8, 3964 MatrixIndexGPR32Op8_11, 3965 sme_elm_idx0_7, "mov">; 3966 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3967 ZZ_h_mul_r, MatrixOp16, 3968 MatrixIndexGPR32Op8_11, 3969 sme_elm_idx0_7, "mov">; 3970 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3971 ZZ_s_mul_r, MatrixOp32, 3972 MatrixIndexGPR32Op8_11, 3973 sme_elm_idx0_7, "mov">; 3974 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3975 ZZ_d_mul_r, MatrixOp64, 3976 MatrixIndexGPR32Op8_11, 3977 sme_elm_idx0_7, "mov">; 3978 3979 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3980 ZZ_b_mul_r, MatrixOp8, 3981 MatrixIndexGPR32Op8_11, 3982 sme_elm_idx0_7, "mov", "vgx2">; 3983 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3984 ZZ_h_mul_r, MatrixOp16, 3985 MatrixIndexGPR32Op8_11, 3986 sme_elm_idx0_7, "mov", "vgx2">; 3987 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 3988 ZZ_s_mul_r, MatrixOp32, 3989 MatrixIndexGPR32Op8_11, 3990 sme_elm_idx0_7, "mov", "vgx2">; 3991 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), 3992 ZZ_d_mul_r, MatrixOp64, 3993 MatrixIndexGPR32Op8_11, 3994 sme_elm_idx0_7, "mov", "vgx2">; 3995 } 3996} 3997 3998// move array to vector, four registers 3999multiclass sme2_mova_array_to_vec_vg4_multi<bits<4> opc, string mnemonic> { 4000 def NAME : sme2_mova_array_to_vec_vg24_multi<opc, ZZZZ_d_mul_r, MatrixOp64, 4001 mnemonic, "vgx4"> { 4002 bits<3> Zd; 4003 let Inst{4-2} = Zd; 4004 } 4005 4006 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4007 ZZZZ_b_mul_r, MatrixOp8, 4008 MatrixIndexGPR32Op8_11, 4009 sme_elm_idx0_7, mnemonic>; 4010 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4011 ZZZZ_h_mul_r, MatrixOp16, 4012 MatrixIndexGPR32Op8_11, 4013 sme_elm_idx0_7, mnemonic>; 4014 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4015 ZZZZ_s_mul_r, MatrixOp32, 4016 MatrixIndexGPR32Op8_11, 4017 sme_elm_idx0_7, mnemonic>; 4018 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4019 ZZZZ_d_mul_r, MatrixOp64, 4020 MatrixIndexGPR32Op8_11, 4021 sme_elm_idx0_7, mnemonic>; 4022 4023 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4024 ZZZZ_b_mul_r, MatrixOp8, 4025 MatrixIndexGPR32Op8_11, 4026 sme_elm_idx0_7, mnemonic, "vgx4">; 4027 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4028 ZZZZ_h_mul_r, MatrixOp16, 4029 MatrixIndexGPR32Op8_11, 4030 sme_elm_idx0_7, mnemonic, "vgx4">; 4031 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4032 ZZZZ_s_mul_r, MatrixOp32, 4033 MatrixIndexGPR32Op8_11, 4034 sme_elm_idx0_7, mnemonic, "vgx4">; 4035 4036 if !eq(mnemonic, "mova") then { 4037 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4038 ZZZZ_b_mul_r, MatrixOp8, 4039 MatrixIndexGPR32Op8_11, 4040 sme_elm_idx0_7, "mov">; 4041 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4042 ZZZZ_h_mul_r, MatrixOp16, 4043 MatrixIndexGPR32Op8_11, 4044 sme_elm_idx0_7, "mov">; 4045 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4046 ZZZZ_s_mul_r, MatrixOp32, 4047 MatrixIndexGPR32Op8_11, 4048 sme_elm_idx0_7, "mov">; 4049 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4050 ZZZZ_d_mul_r, MatrixOp64, 4051 MatrixIndexGPR32Op8_11, 4052 sme_elm_idx0_7, "mov">; 4053 4054 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4055 ZZZZ_b_mul_r, MatrixOp8, 4056 MatrixIndexGPR32Op8_11, 4057 sme_elm_idx0_7, "mov", "vgx4">; 4058 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4059 ZZZZ_h_mul_r, MatrixOp16, 4060 MatrixIndexGPR32Op8_11, 4061 sme_elm_idx0_7, "mov", "vgx4">; 4062 defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), 4063 ZZZZ_s_mul_r, MatrixOp32, 4064 MatrixIndexGPR32Op8_11, 4065 sme_elm_idx0_7, "mov", "vgx4">; 4066 defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), 4067 ZZZZ_d_mul_r, MatrixOp64, 4068 MatrixIndexGPR32Op8_11, 4069 sme_elm_idx0_7, "mov", "vgx4">; 4070 } 4071} 4072 4073//===----------------------------------------------------------------------===// 4074// SME2 multi-vec saturating shift right narrow 4075class sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> 4076 : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, tvecshiftR16:$imm4), 4077 mnemonic, "\t$Zd, $Zn, $imm4", 4078 "", []>, Sched<[]> { 4079 bits<4> imm4; 4080 bits<4> Zn; 4081 bits<5> Zd; 4082 let Inst{31-21} = 0b11000001111; 4083 let Inst{20} = op; 4084 let Inst{19-16} = imm4; 4085 let Inst{15-10} = 0b110101; 4086 let Inst{9-6} = Zn; 4087 let Inst{5} = u; 4088 let Inst{4-0} = Zd; 4089} 4090 4091multiclass sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u, SDPatternOperator intrinsic> { 4092 def _H : sme2_sat_shift_vector_vg2<mnemonic, op, u>; 4093 4094 def : SME2_Sat_Shift_VG2_Pat<NAME # _H, intrinsic, nxv8i16, nxv4i32, tvecshiftR16>; 4095} 4096 4097class sme2_sat_shift_vector_vg4<bits<2> sz, bits<3> op, ZPRRegOp zpr_ty, 4098 RegisterOperand vector_ty, Operand imm_ty, 4099 string mnemonic> 4100 : I<(outs zpr_ty:$Zd), (ins vector_ty:$Zn, imm_ty:$imm), 4101 mnemonic, "\t$Zd, $Zn, $imm", 4102 "", []>, Sched<[]> { 4103 bits<3> Zn; 4104 bits<5> Zd; 4105 let Inst{31-24} = 0b11000001; 4106 let Inst{23-22} = sz; 4107 let Inst{21} = 0b1; 4108 // Inst{20-16} = imm5; 4109 let Inst{15-11} = 0b11011; 4110 let Inst{10} = op{2}; 4111 let Inst{9-7} = Zn; 4112 let Inst{6-5} = op{1-0}; 4113 let Inst{4-0} = Zd; 4114} 4115 4116multiclass sme2_sat_shift_vector_vg4<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { 4117 def _B : sme2_sat_shift_vector_vg4<{0,1}, op, ZPR8, ZZZZ_s_mul_r, tvecshiftR32, 4118 mnemonic>{ 4119 bits<5> imm; 4120 let Inst{20-16} = imm; 4121 } 4122 def _H : sme2_sat_shift_vector_vg4<{1,?}, op, ZPR16, ZZZZ_d_mul_r, tvecshiftR64, 4123 mnemonic> { 4124 bits<6> imm; 4125 let Inst{22} = imm{5}; 4126 let Inst{20-16} = imm{4-0}; 4127 } 4128 4129 def : SME2_Sat_Shift_VG4_Pat<NAME # _B, intrinsic, nxv16i8, nxv4i32, tvecshiftR32>; 4130 def : SME2_Sat_Shift_VG4_Pat<NAME # _H, intrinsic, nxv8i16, nxv2i64, tvecshiftR64>; 4131} 4132 4133//===----------------------------------------------------------------------===// 4134// SME2 Multi-vector - SVE Select 4135class sme2_sel_vector_vg24<bits<2> sz, bits<4> op, RegisterOperand vector_ty, 4136 string mnemonic> 4137 : I<(outs vector_ty:$Zd), 4138 (ins PNRAny_p8to15:$PNg, vector_ty:$Zn, vector_ty:$Zm), 4139 mnemonic, "\t$Zd, $PNg, $Zn, $Zm", 4140 "", []>, Sched<[]> { 4141 bits<3> PNg; 4142 let Inst{31-24} = 0b11000001; 4143 let Inst{23-22} = sz; 4144 let Inst{21} = 0b1; 4145 let Inst{17-16} = op{3-2}; 4146 let Inst{15-13} = 0b100; 4147 let Inst{12-10} = PNg; 4148 let Inst{6} = op{1}; 4149 let Inst{5} = 0b0; 4150 let Inst{1} = op{0}; 4151 let Inst{0} = 0b0; 4152} 4153 4154class sme2_sel_vector_vg2<bits<2> sz, RegisterOperand vector_ty, 4155 string mnemonic> 4156 : sme2_sel_vector_vg24<sz, {?,0,?,?}, vector_ty, mnemonic> { 4157 bits<4> Zm; 4158 bits<4> Zn; 4159 bits<4> Zd; 4160 let Inst{20-17} = Zm; 4161 let Inst{9-6} = Zn; 4162 let Inst{4-1} = Zd; 4163} 4164 4165multiclass sme2_sel_vector_vg2<string mnemonic>{ 4166 def _B : sme2_sel_vector_vg2<0b00, ZZ_b_mul_r, mnemonic>; 4167 def _H : sme2_sel_vector_vg2<0b01, ZZ_h_mul_r, mnemonic>; 4168 def _S : sme2_sel_vector_vg2<0b10, ZZ_s_mul_r, mnemonic>; 4169 def _D : sme2_sel_vector_vg2<0b11, ZZ_d_mul_r, mnemonic>; 4170} 4171class sme2_sel_vector_vg4<bits<2> sz, RegisterOperand vector_ty, 4172 string mnemonic> 4173 : sme2_sel_vector_vg24<sz, 0b0100, vector_ty, mnemonic> { 4174 bits<3> Zm; 4175 bits<3> Zn; 4176 bits<3> Zd; 4177 let Inst{20-18} = Zm; 4178 let Inst{9-7} = Zn; 4179 let Inst{4-2} = Zd; 4180} 4181multiclass sme2_sel_vector_vg4<string mnemonic> { 4182 def _B : sme2_sel_vector_vg4<0b00, ZZZZ_b_mul_r, mnemonic>; 4183 def _H : sme2_sel_vector_vg4<0b01, ZZZZ_h_mul_r, mnemonic>; 4184 def _S : sme2_sel_vector_vg4<0b10, ZZZZ_s_mul_r, mnemonic>; 4185 def _D : sme2_sel_vector_vg4<0b11, ZZZZ_d_mul_r, mnemonic>; 4186} 4187 4188//===----------------------------------------------------------------------===// 4189// Non contiguous Load and Store 4190 4191class sme2_ld_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, 4192 RegisterOperand multi_vector_ty, 4193 RegisterOperand gpr_ty, 4194 string mnemonic> 4195 : I<(outs multi_vector_ty:$Zt), 4196 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4197 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 4198 "", []>, Sched<[]> { 4199 bits<5> Rm; 4200 bits<3> PNg; 4201 bits<5> Rn; 4202 bits<4> Zt; 4203 let Inst{31-21} = 0b10100001000; 4204 let Inst{20-16} = Rm; 4205 let Inst{15} = 0b0; 4206 let Inst{14-13} = msz; 4207 let Inst{12-10} = PNg; 4208 let Inst{9-5} = Rn; 4209 let Inst{4} = Zt{3}; 4210 let Inst{3} = n; 4211 let Inst{2-0} = Zt{2-0}; 4212 4213 let mayLoad = 1; 4214} 4215 4216class sme2_ld_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, 4217 RegisterOperand multi_vector_ty, 4218 RegisterOperand gpr_ty, 4219 string mnemonic> 4220 : I<(outs multi_vector_ty:$Zt), 4221 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4222 mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", 4223 "", []>, Sched<[]> { 4224 bits<5> Rm; 4225 bits<3> PNg; 4226 bits<5> Rn; 4227 bits<3> Zt; 4228 let Inst{31-21} = 0b10100001000; 4229 let Inst{20-16} = Rm; 4230 let Inst{15} = 0b1; 4231 let Inst{14-13} = msz; 4232 let Inst{12-10} = PNg; 4233 let Inst{9-5} = Rn; 4234 let Inst{4} = Zt{2}; 4235 let Inst{3} = n; 4236 let Inst{2} = 0b0; 4237 let Inst{1-0} = Zt{1-0}; 4238 4239 let mayLoad = 1; 4240} 4241 4242class sme2_ld_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, 4243 RegisterOperand multi_vector_ty, 4244 Operand index_ty, 4245 string mnemonic> 4246 : I<(outs multi_vector_ty:$Zt), 4247 (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), 4248 mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]", 4249 "", []>, Sched<[]> { 4250 bits<4> imm4; 4251 bits<3> PNg; 4252 bits<5> Rn; 4253 let Inst{31-20} = 0b101000010100; 4254 let Inst{19-16} = imm4; 4255 let Inst{15} = op{1}; 4256 let Inst{14-13} = msz; 4257 let Inst{12-10} = PNg; 4258 let Inst{9-5} = Rn; 4259 let Inst{3} = n; 4260 let Inst{2} = op{0}; 4261 4262 let mayLoad = 1; 4263} 4264 4265multiclass sme2_ld_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, 4266 RegisterOperand multi_vector_ty, 4267 Operand index_ty, 4268 string mnemonic>{ 4269 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, 4270 multi_vector_ty, 4271 index_ty, mnemonic> { 4272 bits<4> Zt; 4273 let Inst{4} = Zt{3}; 4274 let Inst{2-0} = Zt{2-0}; 4275 } 4276 4277 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", 4278 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 4279} 4280 4281multiclass sme2_ld_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, 4282 RegisterOperand multi_vector_ty, 4283 Operand index_ty, 4284 string mnemonic> { 4285 def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, 0b10, 4286 multi_vector_ty, 4287 index_ty, mnemonic> { 4288 bits<3> Zt; 4289 let Inst{4} = Zt{2}; 4290 let Inst{1-0} = Zt{1-0}; 4291 } 4292 4293 def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", 4294 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; 4295} 4296 4297//===----------------------------------------------------------------------===// 4298// SME2 Non-Contiguous Store 4299class sme2_st_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, 4300 RegisterOperand multi_vector_ty, 4301 RegisterOperand gpr_ty, 4302 string mnemonic> 4303 : I<(outs ), 4304 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4305 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 4306 "", []>, Sched<[]> { 4307 bits<5> Rm; 4308 bits<3> PNg; 4309 bits<5> Rn; 4310 bits<4> Zt; 4311 let Inst{31-21} = 0b10100001001; 4312 let Inst{20-16} = Rm; 4313 let Inst{15} = 0b0; 4314 let Inst{14-13} = msz; 4315 let Inst{12-10} = PNg; 4316 let Inst{9-5} = Rn; 4317 let Inst{4} = Zt{3}; 4318 let Inst{3} = n; 4319 let Inst{2-0} = Zt{2-0}; 4320 4321 let mayStore = 1; 4322} 4323 4324class sme2_st_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, 4325 RegisterOperand multi_vector_ty, 4326 RegisterOperand gpr_ty, 4327 string mnemonic> 4328 : I<(outs ), 4329 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), 4330 mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", 4331 "", []>, Sched<[]> { 4332 bits<5> Rm; 4333 bits<3> PNg; 4334 bits<5> Rn; 4335 bits<3> Zt; 4336 let Inst{31-21} = 0b10100001001; 4337 let Inst{20-16} = Rm; 4338 let Inst{15} = 0b1; 4339 let Inst{14-13} = msz; 4340 let Inst{12-10} = PNg; 4341 let Inst{9-5} = Rn; 4342 let Inst{4} = Zt{2}; 4343 let Inst{3} = n; 4344 let Inst{2} = 0b0; 4345 let Inst{1-0} = Zt{1-0}; 4346 4347 let mayStore = 1; 4348} 4349 4350class sme2_st_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, 4351 RegisterOperand multi_vector_ty, 4352 Operand index_ty, 4353 string mnemonic> 4354 : I<(outs ), 4355 (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), 4356 mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]", 4357 "", []>, Sched<[]> { 4358 bits<4> imm4; 4359 bits<3> PNg; 4360 bits<5> Rn; 4361 let Inst{31-20} = 0b101000010110; 4362 let Inst{19-16} = imm4; 4363 let Inst{15} = op{1}; 4364 let Inst{14-13} = msz; 4365 let Inst{12-10} = PNg; 4366 let Inst{9-5} = Rn; 4367 let Inst{3} = n; 4368 let Inst{2} = op{0}; 4369 4370 let mayStore = 1; 4371} 4372 4373 4374multiclass sme2_st_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, 4375 RegisterOperand multi_vector_ty, 4376 Operand index_ty, 4377 string mnemonic> { 4378 def NAME: sme2_st_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, 4379 multi_vector_ty, 4380 index_ty, mnemonic> { 4381 bits<4> Zt; 4382 let Inst{4} = Zt{3}; 4383 let Inst{2-0} = Zt{2-0}; 4384 } 4385 4386 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", 4387 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 4388} 4389 4390multiclass sme2_st_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, 4391 RegisterOperand multi_vector_ty, 4392 Operand index_ty, 4393 string mnemonic> { 4394 def NAME : sme2_st_vector_vg24_multi_scalar_immediate<msz, n, 0b10, 4395 multi_vector_ty, 4396 index_ty, mnemonic> { 4397 bits<3> Zt; 4398 let Inst{4} = Zt{2}; 4399 let Inst{1-0} = Zt{1-0}; 4400 } 4401 4402 def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", 4403 (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; 4404} 4405 4406//===----------------------------------------------------------------------===// 4407// SME2.1 4408//===----------------------------------------------------------------------===// 4409// SME zeroing move array to vector 4410class sme2p1_movaz_tile_to_vec_base<bits<2> sz, bit q, bit v, ZPRRegOp vector_ty, 4411 RegisterOperand tile_ty, Operand index_ty, 4412 string mnemonic> 4413 : I<(outs vector_ty:$Zd, tile_ty:$ZAn), 4414 (ins tile_ty:$_ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), 4415 mnemonic, "\t$Zd, $ZAn[$Rs, $imm]", 4416 "", []>, Sched<[]> { 4417 bits<2> Rs; 4418 bits<5> Zd; 4419 let Inst{31-24} = 0b11000000; 4420 let Inst{23-22} = sz; 4421 let Inst{21-17} = 0b00001; 4422 let Inst{16} = q; 4423 let Inst{15} = v; 4424 let Inst{14-13} = Rs; 4425 let Inst{12-9} = 0b0001; 4426 let Inst{4-0} = Zd; 4427 let Constraints = "$ZAn = $_ZAn"; 4428} 4429 4430multiclass sme2p1_movaz_tile_to_vec_base<bit v, string mnemonic> { 4431 def _B : sme2p1_movaz_tile_to_vec_base<0b00, 0b0, v, ZPR8, 4432 !if(v, TileVectorOpV8, TileVectorOpH8), 4433 sme_elm_idx0_15, mnemonic> { 4434 bits<4> imm; 4435 let Inst{8-5} = imm; 4436 } 4437 4438 def _H : sme2p1_movaz_tile_to_vec_base<0b01, 0b0, v, ZPR16, 4439 !if(v, TileVectorOpV16, TileVectorOpH16), 4440 sme_elm_idx0_7, mnemonic> { 4441 bits<1> ZAn; 4442 bits<3> imm; 4443 let Inst{8} = ZAn; 4444 let Inst{7-5} = imm; 4445 } 4446 4447 def _S : sme2p1_movaz_tile_to_vec_base<0b10, 0b0, v, ZPR32, 4448 !if(v, TileVectorOpV32, TileVectorOpH32), 4449 sme_elm_idx0_3, mnemonic> { 4450 bits<2> ZAn; 4451 bits<2> imm; 4452 let Inst{8-7} = ZAn; 4453 let Inst{6-5} = imm; 4454 } 4455 4456 def _D : sme2p1_movaz_tile_to_vec_base<0b11, 0b0, v, ZPR64, 4457 !if(v, TileVectorOpV64, TileVectorOpH64), 4458 sme_elm_idx0_1, mnemonic> { 4459 bits<3> ZAn; 4460 bits<1> imm; 4461 let Inst{8-6} = ZAn; 4462 let Inst{5} = imm; 4463 } 4464 4465 def _Q : sme2p1_movaz_tile_to_vec_base<0b11, 0b1, v, ZPR128, 4466 !if(v, TileVectorOpV128, TileVectorOpH128), 4467 sme_elm_idx0_0, mnemonic> { 4468 bits<4> ZAn; 4469 let Inst{8-5} = ZAn; 4470 } 4471} 4472 4473multiclass sme2p1_movaz_tile_to_vec<string mnemonic>{ 4474 defm _H : sme2p1_movaz_tile_to_vec_base<0b0, mnemonic>; 4475 defm _V : sme2p1_movaz_tile_to_vec_base<0b1, mnemonic>; 4476} 4477 4478//===----------------------------------------------------------------------===// 4479// SME2.1 multiple vectors zero array 4480 4481class sme2p1_zero_matrix<bits<6> opc, Operand index_ty, string mnemonic, 4482 string vg_acronym=""> 4483 : I<(outs MatrixOp64:$ZAd), 4484 (ins MatrixOp64:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm), 4485 mnemonic, "\t$ZAd[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", 4486 "", []>, Sched<[]> { 4487 bits <2> Rv; 4488 let Inst{31-18} = 0b11000000000011; 4489 let Inst{17-15} = opc{5-3}; 4490 let Inst{14-13} = Rv; 4491 let Inst{12-3} = 0b0000000000; 4492 let Inst{2-0} = opc{2-0}; 4493 let Constraints = "$ZAd = $_ZAd"; 4494} 4495 4496multiclass sme2p1_zero_matrix<string mnemonic> { 4497 def _VG2_Z : sme2p1_zero_matrix<{0b000,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx2"> { 4498 bits<3> imm; 4499 let Inst{2-0} = imm; 4500 } 4501 def _2Z : sme2p1_zero_matrix<{0b001,?,?,?}, uimm3s2range, mnemonic> { 4502 bits<3> imm; 4503 let Inst{2-0} = imm; 4504 } 4505 def _VG2_2Z : sme2p1_zero_matrix<{0b0100,?,?}, uimm2s2range, mnemonic, "vgx2"> { 4506 bits<2> imm; 4507 let Inst{1-0} = imm; 4508 } 4509 def _VG4_2Z : sme2p1_zero_matrix<{0b0110,?,?}, uimm2s2range, mnemonic, "vgx4"> { 4510 bits<2> imm; 4511 let Inst{1-0} = imm; 4512 } 4513 def _VG4_Z : sme2p1_zero_matrix<{0b100,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx4"> { 4514 bits<3> imm; 4515 let Inst{2-0} = imm; 4516 } 4517 def _4Z : sme2p1_zero_matrix<{0b1010,?,?}, uimm2s4range, mnemonic> { 4518 bits<2> imm; 4519 let Inst{1-0} = imm; 4520 } 4521 def _VG2_4Z :sme2p1_zero_matrix<{0b11000,?}, uimm1s4range, mnemonic, "vgx2"> { 4522 bits<1> imm; 4523 let Inst{0} = imm; 4524 } 4525 def _VG4_4Z :sme2p1_zero_matrix<{0b11100,?}, uimm1s4range, mnemonic, "vgx4"> { 4526 bits<1> imm; 4527 let Inst{0} = imm; 4528 } 4529} 4530 4531//===----------------------------------------------------------------------===// 4532// SME2.1 lookup table expand two non-contiguous registers 4533 4534class sme2p1_luti_vector_vg2_index<bits<4> op, bits<2> sz, RegisterOperand vector_ty, 4535 AsmVectorIndexOpnd index_ty, 4536 string mnemonic> 4537 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 4538 mnemonic, "\t$Zd, $ZTt, $Zn$i", 4539 "", []>, Sched<[]> { 4540 bits<5> Zn; 4541 bits<4> Zd; 4542 let Inst{31-19} = 0b1100000010011; 4543 let Inst{18-15} = op; 4544 let Inst{14} = 0b1; 4545 let Inst{13-12} = sz; 4546 let Inst{11-10} = 0b00; 4547 let Inst{9-5} = Zn; 4548 let Inst{4} = Zd{3}; 4549 let Inst{3} = 0b0; 4550 let Inst{2-0} = Zd{2-0}; 4551} 4552 4553class sme2p1_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 4554 AsmVectorIndexOpnd index_ty, 4555 string mnemonic> 4556 : sme2p1_luti_vector_vg2_index<{1,?,?,?}, sz, vector_ty, index_ty, mnemonic> { 4557 bits<3> i; 4558 let Inst{17-15} = i; 4559} 4560 4561multiclass sme2p1_luti2_vector_vg2_index<string mnemonic> { 4562 def _B : sme2p1_luti2_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexH, 4563 mnemonic>; 4564 def _H : sme2p1_luti2_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexH, 4565 mnemonic>; 4566} 4567 4568class sme2p1_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, 4569 AsmVectorIndexOpnd index_ty, 4570 string mnemonic> 4571 : sme2p1_luti_vector_vg2_index<{0b01,?,?}, sz, vector_ty, index_ty, mnemonic> { 4572 bits<2> i; 4573 let Inst{16-15} = i; 4574} 4575multiclass sme2p1_luti4_vector_vg2_index<string mnemonic> { 4576 def _B : sme2p1_luti4_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexS, 4577 mnemonic>; 4578 def _H : sme2p1_luti4_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexS, 4579 mnemonic>; 4580} 4581 4582// SME2.1 lookup table expand four non-contiguous registers 4583class sme2p1_luti_vector_vg4_index<bits<3> op, bits<2> sz, RegisterOperand vector_ty, 4584 AsmVectorIndexOpnd index_ty, 4585 string mnemonic> 4586 : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), 4587 mnemonic, "\t$Zd, $ZTt, $Zn$i", 4588 "", []>, Sched<[]> { 4589 bits<5> Zn; 4590 bits<3> Zd; 4591 let Inst{31-19} = 0b1100000010011; 4592 let Inst{18-16} = op; 4593 let Inst{15-14} = 0b10; 4594 let Inst{13-12} = sz; 4595 let Inst{11-10} = 0b00; 4596 let Inst{9-5} = Zn; 4597 let Inst{4} = Zd{2}; 4598 let Inst{3-2} = 0b00; 4599 let Inst{1-0} = Zd{1-0}; 4600} 4601 4602class sme2p1_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 4603 AsmVectorIndexOpnd index_ty, 4604 string mnemonic> 4605 : sme2p1_luti_vector_vg4_index<{1,?,?}, sz, vector_ty, index_ty, mnemonic> { 4606 bits<2> i; 4607 let Inst{17-16} = i; 4608} 4609 4610multiclass sme2p1_luti2_vector_vg4_index<string mnemonic> { 4611 def _B : sme2p1_luti2_vector_vg4_index<0b00, ZZZZ_b_strided, VectorIndexS, 4612 mnemonic>; 4613 def _H : sme2p1_luti2_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexS, 4614 mnemonic>; 4615} 4616 4617class sme2p1_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, 4618 AsmVectorIndexOpnd index_ty, 4619 string mnemonic> 4620 : sme2p1_luti_vector_vg4_index<{0b01,?}, sz, vector_ty, index_ty, mnemonic> { 4621 bit i; 4622 let Inst{16} = i; 4623} 4624 4625multiclass sme2p1_luti4_vector_vg4_index<string mnemonic> { 4626 def _H: sme2p1_luti4_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexD, mnemonic>; 4627} 4628