1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFSOffset : ComplexPattern<iPTR, 1, "SelectBUFSOffset">; 16 17def BUFAddrKind { 18 int Offset = 0; 19 int OffEn = 1; 20 int IdxEn = 2; 21 int BothEn = 3; 22 int Addr64 = 4; 23} 24 25class getAddrName<int addrKind> { 26 string ret = 27 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 28 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 29 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 30 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 31 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 32 ""))))); 33} 34 35class MUBUFAddr64Table <bit is_addr64, string Name> { 36 bit IsAddr64 = is_addr64; 37 string OpName = Name; 38} 39 40class MTBUFAddr64Table <bit is_addr64, string Name> { 41 bit IsAddr64 = is_addr64; 42 string OpName = Name; 43} 44 45 46//===----------------------------------------------------------------------===// 47// BUF class (base class for MTBUF and MUBUF pseudos) 48//===----------------------------------------------------------------------===// 49 50class BUF_Pseudo <string opName, dag outs, dag ins, 51 string asmOps, list<dag> pattern=[]> : 52 InstSI<outs, ins, "", pattern>, 53 SIMCInstr<opName, SIEncodingFamily.NONE> { 54 55 let isPseudo = 1; 56 let isCodeGenOnly = 1; 57 let Size = 8; 58 let UseNamedOperandTable = 1; 59 60 string Mnemonic = opName; 61 string AsmOperands = asmOps; 62 63 Instruction Opcode = !cast<Instruction>(NAME); 64 65 66 let VM_CNT = 1; 67 let EXP_CNT = 1; 68 69 let Uses = [EXEC]; 70 let hasSideEffects = 0; 71 let SchedRW = [WriteVMEM]; 72 73 74 75 bits<1> offen = 0; 76 bits<1> idxen = 0; 77 bits<1> addr64 = 0; 78 bits<1> lds = 0; 79 bits<1> has_vdata = !not(lds); 80 bits<1> has_vaddr = 1; 81 bits<1> has_glc = 1; 82 bits<1> has_dlc = 1; 83 bits<1> glc_value = 0; // the value for glc if no such operand 84 bits<1> dlc_value = 0; // the value for dlc if no such operand 85 bits<1> has_srsrc = 1; 86 bits<1> has_soffset = 1; 87 bits<1> has_offset = 1; 88 bits<1> has_slc = 1; 89 bits<1> tfe = 0; 90 bits<4> elements = 0; 91 bits<1> has_sccb = 1; 92 bits<1> sccb_value = 0; 93 bits<1> IsBufferInv = 0; 94} 95 96 97 98//===----------------------------------------------------------------------===// 99// MTBUF classes 100//===----------------------------------------------------------------------===// 101 102class MTBUFGetBaseOpcode<string Op> { 103 string ret = !subst("FORMAT_XY", "FORMAT_X", 104 !subst("FORMAT_XYZ", "FORMAT_X", 105 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 106} 107 108 109class MTBUF_Pseudo <string opName, dag outs, dag ins, 110 string asmOps, list<dag> pattern=[]> : 111 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 112 113 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 114 let MTBUF = 1; 115} 116 117class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 118 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 119 120 let isPseudo = 0; 121 let isCodeGenOnly = 0; 122 123 let VM_CNT = 1; 124 let EXP_CNT = 1; 125 let MTBUF = 1; 126 127 // copy relevant pseudo op flags 128 let UseNamedOperandTable = ps.UseNamedOperandTable; 129 let SubtargetPredicate = ps.SubtargetPredicate; 130 let OtherPredicates = ps.OtherPredicates; 131 let AsmMatchConverter = ps.AsmMatchConverter; 132 let Constraints = ps.Constraints; 133 let DisableEncoding = ps.DisableEncoding; 134 let TSFlags = ps.TSFlags; 135 let SchedRW = ps.SchedRW; 136 let mayLoad = ps.mayLoad; 137 let mayStore = ps.mayStore; 138 let IsAtomicRet = ps.IsAtomicRet; 139 let IsAtomicNoRet = ps.IsAtomicNoRet; 140 let Uses = ps.Uses; 141 let Defs = ps.Defs; 142 143 bits<12> offset; 144 bits<5> cpol; 145 bits<7> format; 146 bits<8> vaddr; 147 bits<10> vdata; 148 bits<7> srsrc; 149 bits<8> soffset; 150 151 bits<4> dfmt = format{3-0}; 152 bits<3> nfmt = format{6-4}; 153 154 // GFX90A+ only: instruction uses AccVGPR for data 155 // Bit supersedes tfe. 156 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 157} 158 159class getMTBUFInsDA<list<RegisterClass> vdataList, 160 list<RegisterClass> vaddrList=[], bit hasRestrictedSOffset> { 161 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 162 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 163 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 164 165 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), 166 (ins SCSrc_b32:$soffset)); 167 168 dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, 169 (ins Offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz)); 170 171 dag Inputs = !if(!empty(vaddrList), 172 NonVaddrInputs, 173 !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 174 dag ret = !if(!empty(vdataList), 175 Inputs, 176 !con((ins vdata_op:$vdata), Inputs)); 177} 178 179class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit hasRestrictedSOffset> { 180 dag ret = 181 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasRestrictedSOffset>.ret, 182 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret, 183 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32], hasRestrictedSOffset>.ret, 184 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret, 185 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasRestrictedSOffset>.ret, 186 (ins)))))); 187} 188 189class getMTBUFAsmOps<int addrKind> { 190 string Pfx = 191 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 192 !if(!eq(addrKind, BUFAddrKind.OffEn), 193 "$vaddr, $srsrc,$format $soffset offen", 194 !if(!eq(addrKind, BUFAddrKind.IdxEn), 195 "$vaddr, $srsrc,$format $soffset idxen", 196 !if(!eq(addrKind, BUFAddrKind.BothEn), 197 "$vaddr, $srsrc,$format $soffset idxen offen", 198 !if(!eq(addrKind, BUFAddrKind.Addr64), 199 "$vaddr, $srsrc,$format $soffset addr64", 200 ""))))); 201 string ret = " $vdata, " # Pfx # "$offset$cpol"; 202} 203 204class MTBUF_SetupAddr<int addrKind> { 205 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 206 !eq(addrKind, BUFAddrKind.BothEn)); 207 208 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 209 !eq(addrKind, BUFAddrKind.BothEn)); 210 211 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 212 213 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 214} 215 216class MTBUF_Load_Pseudo <string opName, 217 int addrKind, 218 RegisterClass vdataClass, 219 int elems, 220 bit hasRestrictedSOffset = 0, 221 list<dag> pattern=[], 222 // Workaround bug bz30254 223 int addrKindCopy = addrKind> 224 : MTBUF_Pseudo<opName, 225 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 226 getMTBUFIns<addrKindCopy, [], hasRestrictedSOffset>.ret, 227 getMTBUFAsmOps<addrKindCopy>.ret, 228 pattern>, 229 MTBUF_SetupAddr<addrKindCopy> { 230 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 231 let mayLoad = 1; 232 let mayStore = 0; 233 let elements = elems; 234} 235 236multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterClass vdataClass, 237 int elems, bit hasRestrictedSOffset> { 238 239 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>, 240 MTBUFAddr64Table<0, NAME>; 241 242 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>, 243 MTBUFAddr64Table<1, NAME>; 244 245 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 246 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 247 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 248 249 let DisableWQM = 1 in { 250 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>; 251 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 252 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 253 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 254 } 255} 256 257multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 258 int elems> { 259 defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>; 260 defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>; 261} 262 263class MTBUF_Store_Pseudo <string opName, 264 int addrKind, 265 RegisterClass vdataClass, 266 int elems, 267 bit hasRestrictedSOffset = 0, 268 list<dag> pattern=[], 269 // Workaround bug bz30254 270 int addrKindCopy = addrKind, 271 RegisterClass vdataClassCopy = vdataClass> 272 : MTBUF_Pseudo<opName, 273 (outs), 274 getMTBUFIns<addrKindCopy, [vdataClassCopy], hasRestrictedSOffset>.ret, 275 getMTBUFAsmOps<addrKindCopy>.ret, 276 pattern>, 277 MTBUF_SetupAddr<addrKindCopy> { 278 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 279 let mayLoad = 0; 280 let mayStore = 1; 281 let elements = elems; 282} 283 284multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterClass vdataClass, 285 int elems, bit hasRestrictedSOffset> { 286 287 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>, 288 MTBUFAddr64Table<0, NAME>; 289 290 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasRestrictedSOffset>, 291 MTBUFAddr64Table<1, NAME>; 292 293 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 294 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 295 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 296 297 let DisableWQM = 1 in { 298 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasRestrictedSOffset>; 299 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasRestrictedSOffset>; 300 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasRestrictedSOffset>; 301 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasRestrictedSOffset>; 302 } 303} 304 305multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 306 int elems> { 307 defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>; 308 defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>; 309} 310 311//===----------------------------------------------------------------------===// 312// MUBUF classes 313//===----------------------------------------------------------------------===// 314 315class MUBUFGetBaseOpcode<string Op> { 316 string ret = !subst("DWORDX2", "DWORD", 317 !subst("DWORDX3", "DWORD", 318 !subst("DWORDX4", "DWORD", Op))); 319} 320 321class MUBUF_Pseudo <string opName, dag outs, dag ins, 322 string asmOps, list<dag> pattern=[]> : 323 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 324 325 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 326 let MUBUF = 1; 327 let AsmMatchConverter = "cvtMubuf"; 328 let usesCustomInserter = 1; 329} 330 331class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 332 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 333 334 let isPseudo = 0; 335 let isCodeGenOnly = 0; 336 337 let VM_CNT = 1; 338 let EXP_CNT = 1; 339 let MUBUF = 1; 340 341 // copy relevant pseudo op flags 342 let SubtargetPredicate = ps.SubtargetPredicate; 343 let AsmMatchConverter = ps.AsmMatchConverter; 344 let OtherPredicates = ps.OtherPredicates; 345 let Constraints = ps.Constraints; 346 let DisableEncoding = ps.DisableEncoding; 347 let TSFlags = ps.TSFlags; 348 let UseNamedOperandTable = ps.UseNamedOperandTable; 349 let SchedRW = ps.SchedRW; 350 let mayLoad = ps.mayLoad; 351 let mayStore = ps.mayStore; 352 let IsAtomicRet = ps.IsAtomicRet; 353 let IsAtomicNoRet = ps.IsAtomicNoRet; 354 let VALU = ps.VALU; 355 let LGKM_CNT = ps.LGKM_CNT; 356 let Uses = ps.Uses; 357 let Defs = ps.Defs; 358 359 bits<12> offset; 360 bits<5> cpol; 361 bits<8> vaddr; 362 bits<10> vdata; 363 bits<7> srsrc; 364 bits<8> soffset; 365 366 // GFX90A+ only: instruction uses AccVGPR for data 367 // Bit supersedes tfe. 368 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 369} 370 371// For cache invalidation instructions. 372class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 373 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 374 375 let AsmMatchConverter = ""; 376 377 let hasSideEffects = 1; 378 let mayLoad = 0; 379 let mayStore = 0; 380 381 let IsBufferInv = 1; 382 // Set everything else to 0. 383 let offen = 0; 384 let idxen = 0; 385 let addr64 = 0; 386 let has_vdata = 0; 387 let has_vaddr = 0; 388 let has_glc = 0; 389 let has_dlc = 0; 390 let glc_value = 0; 391 let dlc_value = 0; 392 let has_srsrc = 0; 393 let has_soffset = 0; 394 let has_offset = 0; 395 let has_slc = 0; 396 let has_sccb = 0; 397 let sccb_value = 0; 398} 399 400class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> { 401 RegisterOperand tfeVDataOp = 402 !cond(!eq(RC.Size, 32) : AVLdSt_64, 403 !eq(RC.Size, 64) : AVLdSt_96, 404 !eq(RC.Size, 96) : AVLdSt_128, 405 !eq(RC.Size, 128) : AVLdSt_160); 406 407 RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret); 408} 409 410class getMUBUFInsDA<list<RegisterClass> vdataList, 411 list<RegisterClass> vaddrList, bit isTFE, bit hasRestrictedSOffset> { 412 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 413 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 414 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret; 415 416 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 417 dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, (ins Offset:$offset, CPol_0:$cpol, i1imm_0:$swz)); 418 419 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 420 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 421} 422 423class getMUBUFElements<ValueType vt> { 424 int ret = 425 !if(!eq(vt, f16), 1, 426 !if(!eq(vt, v2f16), 2, 427 !if(!eq(vt, v3f16), 3, 428 !if(!eq(vt, v4f16), 4, 429 !if(!eq(vt.Size, 32), 1, 430 !if(!eq(vt.Size, 64), 2, 431 !if(!eq(vt.Size, 96), 3, 432 !if(!eq(vt.Size, 128), 4, 0) 433 ) 434 ) 435 ) 436 ) 437 ) 438 ) 439 ); 440} 441 442class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE, bit hasRestrictedSOffset> { 443 dag ret = 444 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasRestrictedSOffset>.ret, 445 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret, 446 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasRestrictedSOffset>.ret, 447 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasRestrictedSOffset>.ret, 448 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasRestrictedSOffset>.ret, 449 (ins)))))); 450} 451 452class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> { 453 string Vdata = !if(noVdata, " ", " $vdata, "); 454 string Lds = !if(isLds, " lds", ""); 455 string TFE = !if(isTFE, " tfe", ""); 456 string MainArgs = 457 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 458 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 459 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 460 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 461 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 462 ""))))); 463 string Offset = "$offset"; 464 string OtherArgs = "$cpol"; 465 466 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE; 467} 468 469class MUBUF_SetupAddr<int addrKind> { 470 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 471 !eq(addrKind, BUFAddrKind.BothEn)); 472 473 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 474 !eq(addrKind, BUFAddrKind.BothEn)); 475 476 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 477 478 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 479} 480 481class MUBUF_Load_Pseudo <string opName, 482 int addrKind, 483 ValueType vdata_vt, 484 bit HasTiedDest = 0, 485 bit isLds = 0, 486 bit isLdsOpc = 0, 487 bit isTFE = 0, 488 bit hasRestrictedSOffset = 0, 489 list<dag> pattern=[], 490 // Workaround bug bz30254 491 int addrKindCopy = addrKind, 492 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret.RegClass, 493 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret> 494 : MUBUF_Pseudo<opName, 495 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 496 !con(getMUBUFIns<addrKindCopy, [], isTFE, hasRestrictedSOffset>.ret, 497 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 498 getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret, 499 pattern>, 500 MUBUF_SetupAddr<addrKindCopy> { 501 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") # 502 "_" # getAddrName<addrKindCopy>.ret; 503 let AsmMatchConverter = "cvtMubuf"; 504 505 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 506 let LGKM_CNT = isLds; 507 let has_vdata = !not(!or(isLds, isLdsOpc)); 508 let mayLoad = 1; 509 let mayStore = isLds; 510 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 511 let tfe = isTFE; 512 let lds = isLds; 513 let elements = getMUBUFElements<vdata_vt>.ret; 514 let VALU = isLds; 515} 516 517class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : GCNPat < 518 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 519 (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset)) 520>; 521 522class MUBUF_Addr64_Load_Pat <Instruction inst, 523 ValueType load_vt = i32, 524 SDPatternOperator ld = null_frag> : GCNPat < 525 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 526 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)) 527>; 528 529multiclass MUBUF_Pseudo_Load_Pats_Common<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 530 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 531 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 532} 533 534multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag>{ 535 let OtherPredicates = [HasUnrestrictedSOffset] in { 536 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst, load_vt, ld>; 537 } 538 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst # "_VBUFFER", load_vt, ld>; 539} 540 541multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt, 542 bit TiedDest, bit isLds, bit isTFE, bit hasRestrictedSOffset> { 543 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 544 545 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>, 546 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 547 548 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>, 549 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 550 551 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 552 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 553 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 554 555 let DisableWQM = 1 in { 556 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 557 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 558 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 559 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasRestrictedSOffset>; 560 } 561} 562 563multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32, 564 bit TiedDest = 0, bit isLds = 0> { 565 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>; 566 defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>; 567 568 if !not(isLds) then { 569 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>; 570 defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>; 571 } 572} 573 574multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 575 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 576 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 577} 578 579multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 580 ValueType load_vt = i32, 581 bit TiedDest = 0, 582 bit isLds = 0, 583 bit isLdsOpc = 1> { 584 585 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 586 587 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 588 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 589 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 590 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 591 592 def _VBUFFER_OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 593 def _VBUFFER_OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 594 def _VBUFFER_IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 595 def _VBUFFER_BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 596} 597 598class MUBUF_Store_Pseudo <string opName, 599 int addrKind, 600 ValueType store_vt, 601 bit isTFE = 0, 602 bit hasRestrictedSOffset = 0, 603 list<dag> pattern=[], 604 // Workaround bug bz30254 605 int addrKindCopy = addrKind> 606 : MUBUF_Pseudo<opName, 607 (outs), 608 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret.RegClass], isTFE, hasRestrictedSOffset>.ret, 609 getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret, 610 pattern>, 611 MUBUF_SetupAddr<addrKindCopy> { 612 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") # 613 getAddrName<addrKindCopy>.ret; 614 let mayLoad = 0; 615 let mayStore = 1; 616 let elements = getMUBUFElements<store_vt>.ret; 617 let tfe = isTFE; 618} 619 620multiclass MUBUF_Pseudo_Store_Pats_Common<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 621 622 def : GCNPat < 623 (st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 624 (!cast<MUBUF_Pseudo>(BaseInst # _OFFSET) store_vt:$vdata, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 625 626 def : GCNPat < 627 (st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 628 (!cast<MUBUF_Pseudo>(BaseInst # _ADDR64) store_vt:$vdata, i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 629} 630 631multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 632 let OtherPredicates = [HasUnrestrictedSOffset] in { 633 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst, store_vt, st>; 634 } 635 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst # "_VBUFFER", store_vt, st>; 636} 637 638multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt, 639 bit isTFE, bit hasRestrictedSOffset> { 640 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 641 642 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>, 643 MUBUFAddr64Table<0, NAME>; 644 645 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasRestrictedSOffset>, 646 MUBUFAddr64Table<1, NAME>; 647 648 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 649 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 650 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 651 652 let DisableWQM = 1 in { 653 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasRestrictedSOffset>; 654 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 655 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 656 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasRestrictedSOffset>; 657 } 658} 659 660multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> { 661 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>; 662 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>; 663 664 defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>; 665 defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>; 666} 667 668class MUBUF_Pseudo_Store_Lds<string opName> 669 : MUBUF_Pseudo<opName, 670 (outs), 671 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset, CPol:$cpol, i1imm:$swz), 672 " $srsrc, $soffset$offset lds$cpol"> { 673 let LGKM_CNT = 1; 674 let mayLoad = 1; 675 let mayStore = 1; 676 677 let has_vdata = 0; 678 let has_vaddr = 0; 679 let lds = 1; 680 let VALU = 1; 681 682 let Uses = [EXEC, M0]; 683 let AsmMatchConverter = "cvtMubuf"; 684} 685 686class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, bit hasRestrictedSOffset, 687 list<RegisterClass> vaddrList=[]> { 688 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 689 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 690 691 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); 692 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); 693 dag SOffset = !if(hasRestrictedSOffset, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 694 dag MainInputs = !con((ins SReg_128:$srsrc), SOffset, (ins Offset:$offset)); 695 dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol), 696 (ins CPol_NonGLC_WithDefault:$cpol)); 697 698 dag ret = !con(Data, MainInputs, CPol); 699} 700 701class getMUBUFAtomicIns<int addrKind, 702 RegisterClass vdataClass, 703 bit vdata_in, 704 bit hasRestrictedSOffset, 705 // Workaround bug bz30254 706 RegisterClass vdataClassCopy=vdataClass> { 707 dag ret = 708 !if(!eq(addrKind, BUFAddrKind.Offset), 709 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset>.ret, 710 !if(!eq(addrKind, BUFAddrKind.OffEn), 711 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret, 712 !if(!eq(addrKind, BUFAddrKind.IdxEn), 713 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VGPR_32]>.ret, 714 !if(!eq(addrKind, BUFAddrKind.BothEn), 715 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VReg_64]>.ret, 716 !if(!eq(addrKind, BUFAddrKind.Addr64), 717 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasRestrictedSOffset, [VReg_64]>.ret, 718 (ins)))))); 719} 720 721class MUBUF_Atomic_Pseudo<string opName, 722 int addrKind, 723 dag outs, 724 dag ins, 725 string asmOps, 726 list<dag> pattern=[], 727 // Workaround bug bz30254 728 int addrKindCopy = addrKind> 729 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 730 MUBUF_SetupAddr<addrKindCopy> { 731 let mayStore = 1; 732 let mayLoad = 1; 733 let hasPostISelHook = 1; 734 let hasSideEffects = 1; 735 let DisableWQM = 1; 736 let has_glc = 0; 737 let has_dlc = 0; 738 let has_sccb = 1; 739 let AsmMatchConverter = "cvtMubufAtomic"; 740} 741 742class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 743 RegisterClass vdataClass, 744 bit hasRestrictedSOffset = 0, 745 list<dag> pattern=[], 746 // Workaround bug bz30254 747 int addrKindCopy = addrKind, 748 RegisterClass vdataClassCopy = vdataClass> 749 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 750 (outs), 751 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0, hasRestrictedSOffset>.ret, 752 getMUBUFAsmOps<addrKindCopy>.ret, 753 pattern> { 754 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 755 let glc_value = 0; 756 let dlc_value = 0; 757 let sccb_value = 0; 758 let IsAtomicNoRet = 1; 759} 760 761class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 762 RegisterClass vdataClass, 763 bit hasRestrictedSOffset = 0, 764 list<dag> pattern=[], 765 // Workaround bug bz30254 766 int addrKindCopy = addrKind, 767 RegisterClass vdataClassCopy = vdataClass, 768 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 769 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 770 (outs vdata_op:$vdata), 771 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1, hasRestrictedSOffset>.ret, 772 getMUBUFAsmOps<addrKindCopy>.ret, 773 pattern> { 774 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 775 let glc_value = 1; 776 let dlc_value = 0; 777 let sccb_value = 0; 778 let IsAtomicRet = 1; 779 let Constraints = "$vdata = $vdata_in"; 780 let DisableEncoding = "$vdata_in"; 781} 782 783multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 784 RegisterClass vdataClass, 785 ValueType vdataType> { 786 let FPAtomic = vdataType.isFP in { 787 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>, 788 MUBUFAddr64Table <0, NAME>; 789 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>, 790 MUBUFAddr64Table <1, NAME>; 791 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 792 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 793 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 794 795 def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>, 796 MUBUFAddr64Table <0, NAME # "_VBUFFER">; 797 def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>, 798 MUBUFAddr64Table <1, NAME # "_VBUFFER">; 799 def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 800 def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 801 def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 802 } 803} 804 805multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 806 RegisterClass vdataClass, 807 ValueType vdataType, 808 SDPatternOperator atomic> { 809 let FPAtomic = vdataType.isFP in { 810 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0, 811 [(set vdataType:$vdata, 812 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 813 vdataType:$vdata_in))]>, 814 MUBUFAddr64Table <0, NAME # "_RTN">; 815 816 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0, 817 [(set vdataType:$vdata, 818 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 819 vdataType:$vdata_in))]>, 820 MUBUFAddr64Table <1, NAME # "_RTN">; 821 822 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 823 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 824 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 825 826 def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1, 827 [(set vdataType:$vdata, 828 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 829 vdataType:$vdata_in))]>, 830 MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">; 831 832 def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1, 833 [(set vdataType:$vdata, 834 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 835 vdataType:$vdata_in))]>, 836 MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">; 837 838 def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 839 def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 840 def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 841 } 842} 843 844multiclass MUBUF_Pseudo_Atomics <string opName, 845 RegisterClass vdataClass, 846 ValueType vdataType, 847 SDPatternOperator atomic = null_frag> : 848 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 849 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 850 851 852//===----------------------------------------------------------------------===// 853// MUBUF Instructions 854//===----------------------------------------------------------------------===// 855 856defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 857 "buffer_load_format_x", f32 858>; 859defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 860 "buffer_load_format_xy", v2f32 861>; 862defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 863 "buffer_load_format_xyz", v3f32 864>; 865defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 866 "buffer_load_format_xyzw", v4f32 867>; 868defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 869 "buffer_store_format_x", f32 870>; 871defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 872 "buffer_store_format_xy", v2f32 873>; 874defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 875 "buffer_store_format_xyz", v3f32 876>; 877defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 878 "buffer_store_format_xyzw", v4f32 879>; 880 881let OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1 in { 882let TiedSourceNotRead = 1 in { 883 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 884 "buffer_load_format_d16_x", i32 885 >; 886 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 887 "buffer_load_format_d16_xy", v2i32 888 >; 889 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 890 "buffer_load_format_d16_xyz", v3i32 891 >; 892 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 893 "buffer_load_format_d16_xyzw", v4i32 894 >; 895} 896 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 897 "buffer_store_format_d16_x", i32 898 >; 899 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 900 "buffer_store_format_d16_xy", v2i32 901 >; 902 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 903 "buffer_store_format_d16_xyz", v3i32 904 >; 905 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 906 "buffer_store_format_d16_xyzw", v4i32 907 >; 908} // End OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1. 909 910let OtherPredicates = [HasPackedD16VMem], D16Buf = 1 in { 911let TiedSourceNotRead = 1 in { 912 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 913 "buffer_load_format_d16_x", f16 914 >; 915 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 916 "buffer_load_format_d16_xy", v2f16 917 >; 918 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 919 "buffer_load_format_d16_xyz", v3f16 920 >; 921 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 922 "buffer_load_format_d16_xyzw", v4f16 923 >; 924} 925 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 926 "buffer_store_format_d16_x", f16 927 >; 928 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 929 "buffer_store_format_d16_xy", v2f16 930 >; 931 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 932 "buffer_store_format_d16_xyz", v3f16 933 >; 934 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 935 "buffer_store_format_d16_xyzw", v4f16 936 >; 937} // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1. 938 939defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 940 "buffer_load_ubyte", i32 941>; 942defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 943 "buffer_load_sbyte", i32 944>; 945defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 946 "buffer_load_ushort", i32 947>; 948defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 949 "buffer_load_sshort", i32 950>; 951defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 952 "buffer_load_dword", i32 953>; 954defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 955 "buffer_load_dwordx2", v2i32 956>; 957defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 958 "buffer_load_dwordx3", v3i32 959>; 960defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 961 "buffer_load_dwordx4", v4i32 962>; 963 964defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 965 "buffer_load_lds_b32", i32 966>; 967defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 968 "buffer_load_lds_format_x", f32 969>; 970defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 971 "buffer_load_lds_i8", i32 972>; 973defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 974 "buffer_load_lds_i16", i32 975>; 976defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 977 "buffer_load_lds_u8", i32 978>; 979defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 980 "buffer_load_lds_u16", i32 981>; 982 983defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 984defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 985defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 986defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 987defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 988defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 989defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 990defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 991defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 992defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 993 994foreach vt = Reg32Types.types in { 995defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>; 996} 997 998foreach vt = VReg_64.RegTypes in { 999defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>; 1000} 1001 1002foreach vt = VReg_96.RegTypes in { 1003defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", vt, load_global>; 1004} 1005 1006foreach vt = VReg_128.RegTypes in { 1007defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", vt, load_global>; 1008} 1009 1010defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 1011 "buffer_store_byte", i32 1012>; 1013defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 1014 "buffer_store_short", i32 1015>; 1016defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 1017 "buffer_store_dword", i32 1018>; 1019defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 1020 "buffer_store_dwordx2", v2i32 1021>; 1022defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 1023 "buffer_store_dwordx3", v3i32 1024>; 1025defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 1026 "buffer_store_dwordx4", v4i32 1027>; 1028 1029defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_BYTE", i32, truncstorei8_global>; 1030defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>; 1031 1032foreach vt = Reg32Types.types in { 1033defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>; 1034} 1035 1036foreach vt = VReg_64.RegTypes in { 1037defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>; 1038} 1039 1040foreach vt = VReg_96.RegTypes in { 1041defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX3", vt, store_global>; 1042} 1043 1044foreach vt = VReg_128.RegTypes in { 1045defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX4", vt, store_global>; 1046} 1047 1048defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 1049 "buffer_atomic_swap", VGPR_32, i32 1050>; 1051defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 1052 "buffer_atomic_cmpswap", VReg_64, v2i32 1053>; 1054defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 1055 "buffer_atomic_add", VGPR_32, i32 1056>; 1057defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 1058 "buffer_atomic_sub", VGPR_32, i32 1059>; 1060defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 1061 "buffer_atomic_smin", VGPR_32, i32 1062>; 1063defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 1064 "buffer_atomic_umin", VGPR_32, i32 1065>; 1066defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 1067 "buffer_atomic_smax", VGPR_32, i32 1068>; 1069defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 1070 "buffer_atomic_umax", VGPR_32, i32 1071>; 1072defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 1073 "buffer_atomic_and", VGPR_32, i32 1074>; 1075defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 1076 "buffer_atomic_or", VGPR_32, i32 1077>; 1078defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1079 "buffer_atomic_xor", VGPR_32, i32 1080>; 1081defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1082 "buffer_atomic_inc", VGPR_32, i32 1083>; 1084defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1085 "buffer_atomic_dec", VGPR_32, i32 1086>; 1087defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1088 "buffer_atomic_swap_x2", VReg_64, i64 1089>; 1090defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1091 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1092>; 1093defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1094 "buffer_atomic_add_x2", VReg_64, i64 1095>; 1096defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1097 "buffer_atomic_sub_x2", VReg_64, i64 1098>; 1099defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1100 "buffer_atomic_smin_x2", VReg_64, i64 1101>; 1102defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1103 "buffer_atomic_umin_x2", VReg_64, i64 1104>; 1105defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1106 "buffer_atomic_smax_x2", VReg_64, i64 1107>; 1108defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1109 "buffer_atomic_umax_x2", VReg_64, i64 1110>; 1111defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1112 "buffer_atomic_and_x2", VReg_64, i64 1113>; 1114defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1115 "buffer_atomic_or_x2", VReg_64, i64 1116>; 1117defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1118 "buffer_atomic_xor_x2", VReg_64, i64 1119>; 1120defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1121 "buffer_atomic_inc_x2", VReg_64, i64 1122>; 1123defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1124 "buffer_atomic_dec_x2", VReg_64, i64 1125>; 1126 1127let OtherPredicates = [HasGFX10_BEncoding] in { 1128 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics < 1129 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1130 >; 1131} 1132 1133let SubtargetPredicate = isGFX8GFX9 in { 1134def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1135} 1136 1137let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1138/* 1139defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1140defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1141*/ 1142 1143def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1144 int_amdgcn_buffer_wbinvl1_sc>; 1145} 1146 1147let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1148 1149defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1150 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1151>; 1152} 1153 1154let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in { 1155defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1156 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1157>; 1158defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1159 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1160>; 1161} 1162 1163let SubtargetPredicate = isGFX6GFX7GFX10 in { 1164defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1165 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1166>; 1167} 1168 1169let SubtargetPredicate = HasD16LoadStore in { 1170let TiedSourceNotRead = 1 in { 1171 1172defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1173 "buffer_load_ubyte_d16", i32, 1 1174>; 1175 1176defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1177 "buffer_load_ubyte_d16_hi", i32, 1 1178>; 1179 1180defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1181 "buffer_load_sbyte_d16", i32, 1 1182>; 1183 1184defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1185 "buffer_load_sbyte_d16_hi", i32, 1 1186>; 1187 1188defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1189 "buffer_load_short_d16", i32, 1 1190>; 1191 1192defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1193 "buffer_load_short_d16_hi", i32, 1 1194>; 1195 1196defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1197 "buffer_load_format_d16_hi_x", i32 1198>; 1199} // End TiedSourceNotRead 1200 1201defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1202 "buffer_store_byte_d16_hi", i32 1203>; 1204 1205defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1206 "buffer_store_short_d16_hi", i32 1207>; 1208 1209defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1210 "buffer_store_format_d16_hi_x", i32 1211>; 1212 1213} // End HasD16LoadStore 1214 1215let SubtargetPredicate = isNotGFX12Plus in 1216def BUFFER_WBINVL1 : MUBUF_Invalidate < 1217 "buffer_wbinvl1", int_amdgcn_buffer_wbinvl1 1218>; 1219 1220let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1221defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1222 "buffer_atomic_add_f32", VGPR_32, f32 1223>; 1224 1225let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in 1226defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1227 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1228>; 1229 1230let SubtargetPredicate = HasAtomicFaddRtnInsts in 1231defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1232 "buffer_atomic_add_f32", VGPR_32, f32, null_frag 1233>; 1234 1235let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in 1236defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1237 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag 1238>; 1239 1240let SubtargetPredicate = isGFX12Plus in { 1241defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Pseudo_Atomics < 1242 "buffer_atomic_cond_sub_u32", VGPR_32, i32 1243>; 1244} 1245 1246let SubtargetPredicate = HasAtomicBufferPkAddBF16Inst in { 1247let FPAtomic = 1 in 1248defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Pseudo_Atomics < 1249 "buffer_atomic_pk_add_bf16", VGPR_32, v2bf16 1250>; 1251} 1252 1253//===----------------------------------------------------------------------===// 1254// MTBUF Instructions 1255//===----------------------------------------------------------------------===// 1256 1257defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1258defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1259defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1260defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1261defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1262defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1263defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1264defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1265 1266let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1267let TiedSourceNotRead = 1 in { 1268 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1269 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1270 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1271 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1272} 1273 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1274 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1275 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1276 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1277} // End HasUnpackedD16VMem. 1278 1279let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1280let TiedSourceNotRead = 1 in { 1281 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1282 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1283 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1284 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1285} 1286 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1287 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1288 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1289 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1290} // End HasPackedD16VMem. 1291 1292let SubtargetPredicate = isGFX7Plus in { 1293 1294//===----------------------------------------------------------------------===// 1295// Instruction definitions for CI and newer. 1296//===----------------------------------------------------------------------===// 1297 1298def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1299 int_amdgcn_buffer_wbinvl1_vol>; 1300 1301} // End let SubtargetPredicate = isGFX7Plus 1302 1303let SubtargetPredicate = isGFX90APlus in { 1304 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1305 let has_glc = 1; 1306 let has_sccb = 1; 1307 let InOperandList = (ins CPol_0:$cpol); 1308 let AsmOperands = "$cpol"; 1309 } 1310 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1311 let SubtargetPredicate = isGFX90AOnly; 1312 } 1313} // End SubtargetPredicate = isGFX90APlus 1314 1315let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in { 1316 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1317} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst 1318 1319let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1320 // Note the names can be buffer_atomic_fmin_x2/buffer_atomic_fmax_x2 1321 // depending on some subtargets. 1322 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1323 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1324} 1325 1326def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1327 let SubtargetPredicate = isGFX940Plus; 1328 let has_glc = 1; 1329 let has_sccb = 1; 1330 let InOperandList = (ins CPol_0:$cpol); 1331 let AsmOperands = "$cpol"; 1332} 1333 1334def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1335def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1336 1337//===----------------------------------------------------------------------===// 1338// MUBUF Patterns 1339//===----------------------------------------------------------------------===// 1340 1341//===----------------------------------------------------------------------===// 1342// buffer_load/store_format patterns 1343//===----------------------------------------------------------------------===// 1344 1345multiclass MUBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1346 string opcode, ValueType memoryVt = vt> { 1347 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1348 1349 def : GCNPat< 1350 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1351 timm:$auxiliary, 0)), 1352 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1353 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1354 >; 1355 1356 def : GCNPat< 1357 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1358 timm:$auxiliary, 0)), 1359 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1360 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1361 >; 1362 1363 def : GCNPat< 1364 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1365 timm:$auxiliary, timm)), 1366 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1367 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1368 >; 1369 1370 def : GCNPat< 1371 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1372 timm:$auxiliary, timm)), 1373 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1374 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1375 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1376 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1377 >; 1378} 1379 1380multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1381 string opcode, ValueType memoryVt = vt>{ 1382 let SubtargetPredicate = HasUnrestrictedSOffset in { 1383 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1384 } 1385 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1386} 1387 1388defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1389defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1390defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1391defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1392defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1393defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1394defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1395defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1396 1397defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">; 1398defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">; 1399defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">; 1400defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">; 1401 1402let OtherPredicates = [HasUnpackedD16VMem] in { 1403 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1404 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1405 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1406 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1407 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1408 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1409} // End HasUnpackedD16VMem. 1410 1411let OtherPredicates = [HasPackedD16VMem] in { 1412 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1413 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1414 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1415 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1416 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1417 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1418 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1419 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1420 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1421} // End HasPackedD16VMem. 1422 1423foreach vt = Reg32Types.types in { 1424defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORD">; 1425} 1426 1427foreach vt = Reg64Types.types in { 1428defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX2">; 1429} 1430 1431foreach vt = Reg96Types.types in { 1432defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX3">; 1433} 1434 1435foreach vt = Reg128Types.types in { 1436defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, vt, "BUFFER_LOAD_DWORDX4">; 1437} 1438 1439defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1440defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1441defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1442defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1443 1444defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v2i32, "BUFFER_LOAD_DWORD_TFE">; 1445defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v3i32, "BUFFER_LOAD_DWORDX2_TFE">; 1446defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v4i32, "BUFFER_LOAD_DWORDX3_TFE">; 1447defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_tfe, v5i32, "BUFFER_LOAD_DWORDX4_TFE">; 1448defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte_tfe, v2i32, "BUFFER_LOAD_SBYTE_TFE">; 1449defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short_tfe, v2i32, "BUFFER_LOAD_SSHORT_TFE">; 1450defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte_tfe, v2i32, "BUFFER_LOAD_UBYTE_TFE">; 1451defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort_tfe, v2i32, "BUFFER_LOAD_USHORT_TFE">; 1452 1453multiclass MUBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1454 string opcode, ValueType memoryVt = vt> { 1455 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1456 1457 def : GCNPat< 1458 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1459 timm:$auxiliary, 0), 1460 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1461 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1462 >; 1463 1464 def : GCNPat< 1465 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1466 timm:$auxiliary, 0), 1467 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1468 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1469 >; 1470 1471 def : GCNPat< 1472 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1473 timm:$auxiliary, timm), 1474 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1475 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1476 >; 1477 1478 def : GCNPat< 1479 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1480 timm:$auxiliary, timm), 1481 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1482 getVregSrcForVT<vt>.ret:$vdata, 1483 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1484 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary), 1485 (extract_swz $auxiliary)) 1486 >; 1487} 1488 1489multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1490 string opcode, ValueType memoryVt = vt> { 1491 let SubtargetPredicate = HasUnrestrictedSOffset in { 1492 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1493 } 1494 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1495} 1496 1497defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1498defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1499defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1500defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1501defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1502defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1503defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1504defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1505defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1506 1507let OtherPredicates = [HasUnpackedD16VMem] in { 1508 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1509 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1510 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1511 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1512 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1513 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1514} // End HasUnpackedD16VMem. 1515 1516let OtherPredicates = [HasPackedD16VMem] in { 1517 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1518 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1519 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1520 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1521 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1522 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1523 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1524 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1525 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1526} // End HasPackedD16VMem. 1527 1528foreach vt = Reg32Types.types in { 1529defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORD">; 1530} 1531 1532foreach vt = Reg64Types.types in { 1533defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX2">; 1534} 1535 1536foreach vt = Reg96Types.types in { 1537defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX3">; 1538} 1539 1540foreach vt = Reg128Types.types in { 1541defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, vt, "BUFFER_STORE_DWORDX4">; 1542} 1543 1544defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1545defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1546 1547//===----------------------------------------------------------------------===// 1548// buffer_atomic patterns 1549//===----------------------------------------------------------------------===// 1550 1551multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1552 foreach RtnMode = ["ret", "noret"] in { 1553 1554 defvar Op = !cast<SDPatternOperator>(OpPrefix 1555 # !if(!eq(RtnMode, "ret"), "", "_noret") 1556 # !if(isIntr, "", "_" # vt)); 1557 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1558 1559 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1560 def : GCNPat< 1561 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)), 1562 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1563 SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset) 1564 >; 1565 1566 def : GCNPat< 1567 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1568 vt:$vdata_in)), 1569 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1570 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, Offset:$offset) 1571 >; 1572 } // end let AddedComplexity 1573 1574 } // end foreach RtnMode 1575} 1576 1577multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1578 let SubtargetPredicate = HasUnrestrictedSOffset in { 1579 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst, isIntr>; 1580 } 1581 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", isIntr>; 1582} 1583 1584multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1585 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1586} 1587 1588multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1589 foreach RtnMode = ["ret", "noret"] in { 1590 1591 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1592 # !if(!eq(RtnMode, "ret"), "", "_noret") 1593 # "_" # vt); 1594 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1595 defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass; 1596 1597 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1598 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1599 data_vt_RC:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1600 Offset:$offset); 1601 def : GCNPat< 1602 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)), 1603 !if(!eq(RtnMode, "ret"), 1604 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, data_vt_RC)), 1605 !if(!eq(vt, i32), sub0, sub0_sub1)), 1606 OffsetResDag) 1607 >; 1608 1609 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1610 data_vt_RC:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1611 SCSrc_b32:$soffset, Offset:$offset); 1612 def : GCNPat< 1613 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1614 data_vt:$vdata_in)), 1615 !if(!eq(RtnMode, "ret"), 1616 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, data_vt_RC)), 1617 !if(!eq(vt, i32), sub0, sub0_sub1)), 1618 Addr64ResDag) 1619 >; 1620 } // end let AddedComplexity 1621 1622 } // end foreach RtnMode 1623} 1624 1625multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1626 let SubtargetPredicate = HasUnrestrictedSOffset in { 1627 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1628 } 1629 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1630} 1631 1632 1633foreach Ty = [i32, i64] in { 1634 1635defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1636 1637defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1638defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1639defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1640defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1641defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1642defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1643defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1644defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1645defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1646defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1647defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1648defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1649 1650} // end foreach Ty 1651 1652let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in { 1653defm : BufferAtomicPat<"atomic_load_fmin_global", f32, "BUFFER_ATOMIC_FMIN">; 1654defm : BufferAtomicPat<"atomic_load_fmax_global", f32, "BUFFER_ATOMIC_FMAX">; 1655} 1656 1657let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1658defm : BufferAtomicPat<"atomic_load_fmin_global", f64, "BUFFER_ATOMIC_MIN_F64">; 1659defm : BufferAtomicPat<"atomic_load_fmax_global", f64, "BUFFER_ATOMIC_MAX_F64">; 1660} 1661 1662defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1663defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1664 1665multiclass SIBufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, 1666 list<string> RtnModes = ["ret", "noret"]> { 1667 foreach RtnMode = RtnModes in { 1668 1669 defvar Op = !cast<SDPatternOperator>(OpPrefix 1670 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1671 1672 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1673 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1674 (extract_cpol_set_glc $auxiliary), (extract_cpol $auxiliary)); 1675 1676 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1677 def : GCNPat< 1678 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1679 timm:$offset, timm:$auxiliary, 0)), 1680 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1681 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1682 timm:$offset, CachePolicy) 1683 >; 1684 1685 def : GCNPat< 1686 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), 1687 timm:$offset, timm:$auxiliary, timm)), 1688 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1689 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1690 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1691 >; 1692 1693 def : GCNPat< 1694 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1695 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, 0)), 1696 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1697 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1698 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1699 >; 1700 1701 def : GCNPat< 1702 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1703 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, timm)), 1704 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1705 getVregSrcForVT<vt>.ret:$vdata_in, 1706 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1707 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1708 >; 1709 } // end let AddedComplexity 1710 1711 } // end foreach RtnMode 1712} 1713 1714multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1715 list<string> RtnModes = ["ret", "noret"]> { 1716 let OtherPredicates = [HasUnrestrictedSOffset] in { 1717 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst, RtnModes>; 1718 } 1719 1720 // FIXME: This needs a !HasUnrestrictedSOffset predicate 1721 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", RtnModes>; 1722} 1723 1724defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1725defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1726defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1727defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1728defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1729defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1730defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1731defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1732defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1733defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1734defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1735defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1736defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1737defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1738defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1739defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1740defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1741defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1742defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1743defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1744defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1745defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1746defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1747defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1748defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1749defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1750 1751let SubtargetPredicate = HasAtomicCSubNoRtnInsts in 1752defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>; 1753 1754let SubtargetPredicate = HasAtomicBufferPkAddBF16Inst in { 1755 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2bf16, "BUFFER_ATOMIC_PK_ADD_BF16">; 1756} 1757 1758let SubtargetPredicate = isGFX12Plus in { 1759 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["ret"]>; 1760} 1761 1762let SubtargetPredicate = HasAtomicCSubNoRtnInsts in { 1763defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>; 1764} 1765 1766let SubtargetPredicate = HasAtomicFMinFMaxF32GlobalInsts in { 1767 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1768 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1769} 1770 1771let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1772 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1773 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1774} 1775 1776class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1777 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1778 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> { 1779 let HasNoUse = true; 1780} 1781 1782multiclass BufferAtomicPatterns_NO_RTN_Common<SDPatternOperator name, ValueType vt, 1783 string opcode> { 1784 def : GCNPat< 1785 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1786 0, (BUFSOffset i32:$soffset), timm:$offset, 1787 timm:$auxiliary, 0), 1788 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1789 timm:$offset, (extract_cpol $auxiliary)) 1790 >; 1791 1792 def : GCNPat< 1793 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1794 0, (BUFSOffset i32:$soffset), timm:$offset, 1795 timm:$auxiliary, timm), 1796 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1797 timm:$offset, (extract_cpol $auxiliary)) 1798 >; 1799 1800 def : GCNPat< 1801 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1802 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1803 timm:$auxiliary, 0), 1804 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1805 timm:$offset, (extract_cpol $auxiliary)) 1806 >; 1807 1808 def : GCNPat< 1809 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1810 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1811 timm:$auxiliary, timm), 1812 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1813 getVregSrcForVT<vt>.ret:$vdata_in, 1814 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1815 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary)) 1816 >; 1817} 1818 1819multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1820 string opcode> { 1821 let SubtargetPredicate = HasUnrestrictedSOffset in { 1822 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode>; 1823 } 1824 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode # "_VBUFFER">; 1825} 1826 1827let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1828 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>; 1829 1830let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in { 1831 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>; 1832} // End SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts 1833 1834let SubtargetPredicate = HasAtomicFaddRtnInsts in 1835 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>; 1836 1837let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in { 1838 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>; 1839} // End SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts 1840 1841let SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst in { 1842 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1843} // End SubtargetPredicate = HasFlatBufferGlobalAtomicFaddF64Inst 1844 1845let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts in { 1846 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1847 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1848} //End let SubtargetPredicate = HasAtomicFMinFMaxF64GlobalInsts 1849 1850multiclass SIBufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1851 foreach RtnMode = ["ret", "noret"] in { 1852 defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1853 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1854 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1855 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1856 (extract_cpol_set_glc $auxiliary), 1857 (extract_cpol $auxiliary)); 1858 defvar SrcRC = getVregSrcForVT<vt>.ret; 1859 defvar DataRC = getVregSrcForVT<data_vt>.ret.RegClass; 1860 defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1); 1861 defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3); 1862 1863 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1864 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1865 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1866 def : GCNPat< 1867 (vt (Op 1868 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1869 timm:$offset, timm:$auxiliary, 0)), 1870 !if(!eq(RtnMode, "ret"), 1871 (EXTRACT_SUBREG OffsetResDag, SubLo), 1872 OffsetResDag) 1873 >; 1874 1875 defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1876 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1877 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1878 CachePolicy); 1879 def : GCNPat< 1880 (vt (Op 1881 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1882 0, (BUFSOffset i32:$soffset), timm:$offset, 1883 timm:$auxiliary, timm)), 1884 !if(!eq(RtnMode, "ret"), 1885 (EXTRACT_SUBREG IdxenResDag, SubLo), 1886 IdxenResDag) 1887 >; 1888 1889 defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1890 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1891 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1892 CachePolicy); 1893 def : GCNPat< 1894 (vt (Op 1895 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 1896 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1897 timm:$auxiliary, 0)), 1898 !if(!eq(RtnMode, "ret"), 1899 (EXTRACT_SUBREG OffenResDag, SubLo), 1900 OffenResDag) 1901 >; 1902 1903 defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1904 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1905 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1906 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1907 def : GCNPat< 1908 (vt (Op 1909 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1910 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1911 timm:$auxiliary, timm)), 1912 !if(!eq(RtnMode, "ret"), 1913 (EXTRACT_SUBREG BothenResDag, SubLo), 1914 BothenResDag) 1915 >; 1916 } // end foreach RtnMode 1917} 1918 1919multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1920 let OtherPredicates = [HasUnrestrictedSOffset] in { 1921 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1922 } 1923 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1924} 1925 1926defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1927defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1928 1929class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1930 PatFrag constant_ld> : GCNPat < 1931 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1932 i32:$offset))), 1933 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1934 >; 1935 1936multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1937 ValueType vt, PatFrag atomic_ld> { 1938 def : GCNPat < 1939 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 1940 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1941 >; 1942 1943 def : GCNPat < 1944 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), 1945 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1946 >; 1947} 1948 1949let SubtargetPredicate = isGFX6GFX7 in { 1950def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1951def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1952def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1953def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1954def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1955def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1956 1957defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1958defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1959} // End SubtargetPredicate = isGFX6GFX7 1960 1961multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt, 1962 PatFrag ld> { 1963 def : GCNPat < 1964 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1965 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $srsrc, $soffset, $offset) 1966 >; 1967} 1968 1969multiclass MUBUFLoad_PatternOffset <string Instr, ValueType vt, 1970 PatFrag ld> { 1971 let OtherPredicates = [HasUnrestrictedSOffset] in { 1972 defm : MUBUFLoad_PatternOffset_Common<Instr, vt, ld>; 1973 } 1974 defm : MUBUFLoad_PatternOffset_Common<Instr # "_VBUFFER", vt, ld>; 1975} 1976 1977let OtherPredicates = [Has16BitInsts] in { 1978 1979defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_constant>; 1980defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_constant>; 1981defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_constant>; 1982defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_global>; 1983defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_global>; 1984defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_global>; 1985 1986defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_USHORT", i16, load_global>; 1987 1988} // End OtherPredicates = [Has16BitInsts] 1989 1990multiclass MUBUFScratchLoadPat_Common <string Instr, 1991 ValueType vt, PatFrag ld> { 1992 def : GCNPat < 1993 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1994 i32:$soffset, i32:$offset))), 1995 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, 0, 0) 1996 >; 1997 1998 def : GCNPat < 1999 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 2000 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, 0, 0) 2001 >; 2002} 2003 2004multiclass MUBUFScratchLoadPat <string Instr, 2005 ValueType vt, PatFrag ld> { 2006 let SubtargetPredicate = HasUnrestrictedSOffset in { 2007 defm : MUBUFScratchLoadPat_Common<Instr, vt, ld>; 2008 } 2009 defm : MUBUFScratchLoadPat_Common<Instr # "_VBUFFER", vt, ld>; 2010} 2011 2012// XXX - Is it possible to have a complex pattern in a PatFrag? 2013multiclass MUBUFScratchLoadPat_D16_Common <string Instr, 2014 ValueType vt, PatFrag ld_frag> { 2015 def : GCNPat < 2016 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in), 2017 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, $in) 2018 >; 2019 2020 def : GCNPat < 2021 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in), 2022 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, $in) 2023 >; 2024} 2025 2026multiclass MUBUFScratchLoadPat_D16 <string Instr, 2027 ValueType vt, PatFrag ld_frag> { 2028 let SubtargetPredicate = HasUnrestrictedSOffset in { 2029 defm : MUBUFScratchLoadPat_D16_Common<Instr, vt, ld_frag>; 2030 } 2031 defm : MUBUFScratchLoadPat_D16_Common<Instr # "_VBUFFER", vt, ld_frag>; 2032} 2033 2034let OtherPredicates = [DisableFlatScratch] in { 2035defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i32, sextloadi8_private>; 2036defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, extloadi8_private>; 2037defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, zextloadi8_private>; 2038defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i16, sextloadi8_private>; 2039defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, extloadi8_private>; 2040defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, zextloadi8_private>; 2041defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SSHORT", i32, sextloadi16_private>; 2042defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, extloadi16_private>; 2043defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, zextloadi16_private>; 2044defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i16, load_private>; 2045 2046foreach vt = Reg32Types.types in { 2047defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORD", vt, load_private>; 2048} 2049defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX2", v2i32, load_private>; 2050defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX3", v3i32, load_private>; 2051defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX4", v4i32, load_private>; 2052 2053let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 2054defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2i16, load_d16_hi_private>; 2055defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2i16, az_extloadi8_d16_hi_private>; 2056defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2i16, sextloadi8_d16_hi_private>; 2057defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2f16, load_d16_hi_private>; 2058defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2f16, az_extloadi8_d16_hi_private>; 2059defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2f16, sextloadi8_d16_hi_private>; 2060 2061defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2i16, load_d16_lo_private>; 2062defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2i16, az_extloadi8_d16_lo_private>; 2063defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2i16, sextloadi8_d16_lo_private>; 2064defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2f16, load_d16_lo_private>; 2065defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2f16, az_extloadi8_d16_lo_private>; 2066defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2f16, sextloadi8_d16_lo_private>; 2067} 2068 2069} // End OtherPredicates = [DisableFlatScratch] 2070 2071multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 2072 ValueType vt, PatFrag atomic_st> { 2073 def : GCNPat < 2074 (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 2075 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 2076 >; 2077 2078 def : GCNPat < 2079 (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)), 2080 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 2081 >; 2082} 2083let SubtargetPredicate = isGFX6GFX7 in { 2084defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 2085defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 2086defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 2087defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 2088defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 2089defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 2090} // End Predicates = isGFX6GFX7 2091 2092 2093multiclass MUBUFStore_PatternOffset_Common <string Instr, ValueType vt, 2094 PatFrag st> { 2095 2096 def : GCNPat < 2097 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 2098 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $vdata, $srsrc, $soffset, $offset) 2099 >; 2100} 2101 2102multiclass MUBUFStore_PatternOffset <string Instr, ValueType vt, 2103 PatFrag st> { 2104 let SubtargetPredicate = HasUnrestrictedSOffset in { 2105 defm : MUBUFStore_PatternOffset_Common<Instr, vt, st>; 2106 } 2107 defm : MUBUFStore_PatternOffset_Common<Instr # "_VBUFFER", vt, st>; 2108} 2109 2110defm : MUBUFStore_PatternOffset <"BUFFER_STORE_BYTE", i16, truncstorei8_global>; 2111defm : MUBUFStore_PatternOffset <"BUFFER_STORE_SHORT", i16, store_global>; 2112 2113multiclass MUBUFScratchStorePat_Common <string Instr, 2114 ValueType vt, PatFrag st, 2115 RegisterClass rc = VGPR_32> { 2116 def : GCNPat < 2117 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 2118 i32:$soffset, i32:$offset)), 2119 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) 2120 >; 2121 2122 def : GCNPat < 2123 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 2124 i32:$offset)), 2125 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) rc:$value, $srsrc, $soffset, $offset, 0, 0) 2126 >; 2127} 2128 2129multiclass MUBUFScratchStorePat <string Instr, 2130 ValueType vt, PatFrag st, 2131 RegisterClass rc = VGPR_32> { 2132 let SubtargetPredicate = HasUnrestrictedSOffset in { 2133 defm : MUBUFScratchStorePat_Common<Instr, vt, st, rc>; 2134 } 2135 defm : MUBUFScratchStorePat_Common<Instr # "_VBUFFER", vt, st, rc>; 2136} 2137 2138let OtherPredicates = [DisableFlatScratch] in { 2139defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i32, truncstorei8_private>; 2140defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i32, truncstorei16_private>; 2141defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i16, truncstorei8_private>; 2142defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i16, store_private>; 2143 2144foreach vt = Reg32Types.types in { 2145defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORD", vt, store_private>; 2146} 2147 2148defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX2", v2i32, store_private, VReg_64>; 2149defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX3", v3i32, store_private, VReg_96>; 2150defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX4", v4i32, store_private, VReg_128>; 2151 2152 2153let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 2154 // Hiding the extract high pattern in the PatFrag seems to not 2155 // automatically increase the complexity. 2156let AddedComplexity = 1 in { 2157defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT_D16_HI", i32, store_hi16_private>; 2158defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE_D16_HI", i32, truncstorei8_hi16_private>; 2159} 2160} 2161} // End OtherPredicates = [DisableFlatScratch] 2162 2163//===----------------------------------------------------------------------===// 2164// MTBUF Patterns 2165//===----------------------------------------------------------------------===// 2166 2167//===----------------------------------------------------------------------===// 2168// tbuffer_load/store_format patterns 2169//===----------------------------------------------------------------------===// 2170 2171multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2172 string opcode, ValueType memoryVt = vt> { 2173 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 2174 2175 def : GCNPat< 2176 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2177 timm:$format, timm:$auxiliary, 0)), 2178 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2179 (as_i8timm $format), 2180 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2181 >; 2182 2183 def : GCNPat< 2184 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2185 timm:$format, timm:$auxiliary, timm)), 2186 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2187 (as_i8timm $format), 2188 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2189 >; 2190 2191 def : GCNPat< 2192 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2193 timm:$format, timm:$auxiliary, 0)), 2194 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2195 (as_i8timm $format), 2196 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2197 >; 2198 2199 def : GCNPat< 2200 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2201 timm:$format, timm:$auxiliary, timm)), 2202 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 2203 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2204 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2205 (as_i8timm $format), 2206 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2207 >; 2208} 2209 2210multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 2211 string opcode, ValueType memoryVt = vt> { 2212 let OtherPredicates = [HasUnrestrictedSOffset] in { 2213 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2214 } 2215 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2216} 2217 2218defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 2219defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 2220defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 2221defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 2222defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 2223defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 2224defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 2225defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 2226 2227let SubtargetPredicate = HasUnpackedD16VMem in { 2228 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2229 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2230 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 2231 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 2232 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 2233} // End HasUnpackedD16VMem. 2234 2235let SubtargetPredicate = HasPackedD16VMem in { 2236 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 2237 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 2238 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 2239 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 2240 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 2241} // End HasPackedD16VMem. 2242 2243multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2244 string opcode, ValueType memoryVt = vt> { 2245 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 2246 2247 def : GCNPat< 2248 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2249 timm:$format, timm:$auxiliary, 0), 2250 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 2251 timm:$offset, (as_i8timm $format), 2252 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2253 >; 2254 2255 def : GCNPat< 2256 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2257 timm:$format, timm:$auxiliary, timm), 2258 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 2259 timm:$offset, (as_i8timm $format), 2260 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2261 >; 2262 2263 def : GCNPat< 2264 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2265 timm:$format, timm:$auxiliary, 0), 2266 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2267 timm:$offset, (as_i8timm $format), 2268 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2269 >; 2270 2271 def : GCNPat< 2272 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), 2273 timm:$offset, timm:$format, timm:$auxiliary, timm), 2274 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2275 getVregSrcForVT<vt>.ret:$vdata, 2276 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2277 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), 2278 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2279 >; 2280} 2281 2282multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 2283 string opcode, ValueType memoryVt = vt> { 2284 let OtherPredicates = [HasUnrestrictedSOffset] in { 2285 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2286 } 2287 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2288} 2289 2290defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2291defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2292defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2293defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2294defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2295defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2296defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2297defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2298 2299let SubtargetPredicate = HasUnpackedD16VMem in { 2300 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2301 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2302 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2303 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2304 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2305} // End HasUnpackedD16VMem. 2306 2307let SubtargetPredicate = HasPackedD16VMem in { 2308 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2309 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2310 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2311 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2312 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2313} // End HasPackedD16VMem. 2314 2315//===----------------------------------------------------------------------===// 2316// Target-specific instruction encodings. 2317//===----------------------------------------------------------------------===// 2318 2319// Shortcut to default Mnemonic from BUF_Pseudo. Hides the cast to the 2320// specific pseudo (bothen in this case) since any of them will work. 2321class get_BUF_ps<string name> { 2322 string Mnemonic = !cast<BUF_Pseudo>(name # "_OFFSET").Mnemonic; 2323} 2324 2325//===----------------------------------------------------------------------===// 2326// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2327//===----------------------------------------------------------------------===// 2328 2329class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2330 string real_name = ps.Mnemonic> : 2331 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2332 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2333 let Inst{31-26} = 0x38; 2334 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2335 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2336 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2337 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2338} 2339 2340multiclass MUBUF_Real_gfx11<bits<8> op, string real_name = !cast<MUBUF_Pseudo>(NAME).Mnemonic> { 2341 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2342 def _gfx11 : Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2343 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2344 // In GFX11 dlc is applicable to all loads/stores/atomics. 2345 let Inst{13} = !if(!or(ps.mayLoad, ps.mayStore), cpol{CPolBit.DLC}, ps.dlc_value); 2346 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2347 let Inst{25-18} = op; 2348 let Inst{53} = ps.tfe; 2349 let Inst{54} = ps.offen; 2350 let Inst{55} = ps.idxen; 2351 let AssemblerPredicate = isGFX11Only; 2352 let DecoderNamespace = "GFX11"; 2353 } 2354} 2355 2356class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef, string asmName> : 2357 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef, asmName> { 2358 let Inst{12} = ps.offen; 2359 let Inst{13} = ps.idxen; 2360 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2361 let Inst{16} = ps.lds; 2362 let Inst{24-18} = op; 2363 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2364 let Inst{55} = ps.tfe; 2365} 2366 2367multiclass MUBUF_Real_gfx10<bits<8> op, string psName = NAME, 2368 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> { 2369 defvar ps = !cast<MUBUF_Pseudo>(psName); 2370 def _gfx10 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10, asmName> { 2371 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2372 let Inst{25} = op{7}; 2373 let AssemblerPredicate = isGFX10Only; 2374 let DecoderNamespace = "GFX10"; 2375 } 2376} 2377 2378multiclass MUBUF_Real_gfx6_gfx7<bits<8> op, string psName = NAME, 2379 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> { 2380 defvar ps = !cast<MUBUF_Pseudo>(psName); 2381 def _gfx6_gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, asmName> { 2382 let Inst{15} = ps.addr64; 2383 let AssemblerPredicate = isGFX6GFX7; 2384 let DecoderNamespace = "GFX6GFX7"; 2385 } 2386} 2387 2388multiclass MUBUF_Real_gfx6<bits<8> op> { 2389 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2390 def _gfx6 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, ps.Mnemonic> { 2391 let Inst{15} = ps.addr64; 2392 let AssemblerPredicate = isGFX6; 2393 let DecoderNamespace = "GFX6"; 2394 } 2395} 2396 2397multiclass MUBUF_Real_gfx7<bits<8> op> { 2398 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2399 def _gfx7 : Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI, ps.Mnemonic> { 2400 let Inst{15} = ps.addr64; 2401 let AssemblerPredicate = isGFX7Only; 2402 let DecoderNamespace = "GFX7"; 2403 } 2404} 2405 2406//===----------------------------------------------------------------------===// 2407// Base ENC_VBUFFER for GFX12. 2408//===----------------------------------------------------------------------===// 2409 2410class VBUFFER_Real <bits<8> op, BUF_Pseudo ps, string real_name> : 2411 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 { 2412 2413 let isPseudo = 0; 2414 let isCodeGenOnly = 0; 2415 2416 let VM_CNT = 1; 2417 let EXP_CNT = 1; 2418 2419 // copy relevant pseudo op flags 2420 let SubtargetPredicate = ps.SubtargetPredicate; 2421 let AsmMatchConverter = ps.AsmMatchConverter; 2422 let OtherPredicates = ps.OtherPredicates; 2423 let Constraints = ps.Constraints; 2424 let DisableEncoding = ps.DisableEncoding; 2425 let TSFlags = ps.TSFlags; 2426 let UseNamedOperandTable = ps.UseNamedOperandTable; 2427 let SchedRW = ps.SchedRW; 2428 let mayLoad = ps.mayLoad; 2429 let mayStore = ps.mayStore; 2430 let IsAtomicRet = ps.IsAtomicRet; 2431 let IsAtomicNoRet = ps.IsAtomicNoRet; 2432 let VALU = ps.VALU; 2433 let LGKM_CNT = ps.LGKM_CNT; 2434 let MUBUF = ps.MUBUF; 2435 let MTBUF = ps.MTBUF; 2436 let Uses = ps.Uses; 2437 let Defs = ps.Defs; 2438 2439 bits<24> offset; 2440 bits<8> vaddr; 2441 bits<10> vdata; 2442 2443 bits<7> srsrc; 2444 bits<7> soffset; 2445 bits<6> cpol; 2446 2447 let Inst{95-72} = !if(ps.has_offset, offset, ?); 2448 let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?); 2449 let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?); 2450 2451 let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?); 2452 let Inst{49-48} = 0b00; 2453 let Inst{6-0} = !if(ps.has_soffset, soffset, ?); 2454 let Inst{21-14} = op; 2455 let Inst{22} = ps.tfe; 2456 let Inst{62} = ps.offen; 2457 let Inst{63} = ps.idxen; 2458 2459 let Inst{54-53} = cpol{2-1}; // th{2-1} 2460 let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0} 2461 let Inst{51-50} = cpol{4-3}; // scope 2462 2463 let Inst{31-26} = 0b110001; 2464} 2465 2466class VBUFFER_Real_gfx12<bits<8> op, BUF_Pseudo ps, string real_name> : 2467 VBUFFER_Real<op, ps, real_name>, 2468 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> { 2469 let AssemblerPredicate = isGFX12Only; 2470 let DecoderNamespace = "GFX12"; 2471} 2472 2473multiclass VBUFFER_MUBUF_Real_gfx12<bits<8> op, string real_name> { 2474 defvar ps = !cast<MUBUF_Pseudo>(NAME); 2475 def _gfx12 : VBUFFER_Real_gfx12<op, ps, real_name> { 2476 // Set the format field to be 1 to avoid round-trip issues, as some tools 2477 // print BUF_FMT_INVALID for format 0. 2478 let Inst{61-55} = 0b0000001; 2479 } 2480 // Have a version of the instruction to disassemble to for any other 2481 // format field values. 2482 def _gfx12_format : VBUFFER_Real<op, ps, real_name> { 2483 let AsmVariantName = "NonParsable"; 2484 let DecoderNamespace = "GFX12"; 2485 } 2486} 2487 2488multiclass VBUFFER_MTBUF_Real_gfx12<bits<4> op, string real_name> { 2489 defvar ps = !cast<MTBUF_Pseudo>(NAME); 2490 def _gfx12 : VBUFFER_Real_gfx12<{0b1000, op}, ps, real_name> { 2491 bits<7> format; 2492 let Inst{61-55} = format; 2493 } 2494} 2495 2496//===----------------------------------------------------------------------===// 2497// MUBUF - GFX11, GFX12. 2498//===----------------------------------------------------------------------===// 2499 2500// gfx11 instruction that accept both old and new assembler name. 2501class Mnem_gfx11_gfx12 <string mnemonic, string real_name> : 2502 AMDGPUMnemonicAlias<mnemonic, real_name> { 2503 let AssemblerPredicate = isGFX11Plus; 2504} 2505 2506class Mnem_gfx11 <string mnemonic, string real_name> : 2507 AMDGPUMnemonicAlias<mnemonic, real_name> { 2508 let AssemblerPredicate = isGFX11Only; 2509} 2510 2511class Mnem_gfx12 <string mnemonic, string real_name> : 2512 AMDGPUMnemonicAlias<mnemonic, real_name> { 2513 let AssemblerPredicate = isGFX12Plus; 2514} 2515 2516multiclass MUBUF_Real_AllAddr_gfx11_Impl2<bits<8> op, string real_name> { 2517 defm _BOTHEN : MUBUF_Real_gfx11<op, real_name>; 2518 defm _IDXEN : MUBUF_Real_gfx11<op, real_name>; 2519 defm _OFFEN : MUBUF_Real_gfx11<op, real_name>; 2520 defm _OFFSET : MUBUF_Real_gfx11<op, real_name>; 2521} 2522 2523multiclass MUBUF_Real_AllAddr_gfx12_Impl2<bits<8> op, string real_name> { 2524 defm _VBUFFER_BOTHEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2525 defm _VBUFFER_IDXEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2526 defm _VBUFFER_OFFEN : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2527 defm _VBUFFER_OFFSET : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2528} 2529 2530multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<bits<8> op, string real_name> : 2531 MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>, 2532 MUBUF_Real_AllAddr_gfx12_Impl2<op, real_name>; 2533 2534multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, bit hasTFE, 2535 string real_name = get_BUF_ps<NAME>.Mnemonic> { 2536 defm NAME : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>; 2537 if hasTFE then 2538 defm _TFE : MUBUF_Real_AllAddr_gfx11_Impl2<op, real_name>; 2539} 2540 2541multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<8> op, bit hasTFE, 2542 string real_name> { 2543 defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>; 2544 if hasTFE then 2545 defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Impl2<op, real_name>; 2546} 2547 2548// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions. 2549multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> : 2550 MUBUF_Real_AllAddr_gfx11_Impl<op, hasTFE>; 2551 2552multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op, 2553 string real_name = get_BUF_ps<NAME>.Mnemonic> : 2554 MUBUF_Real_AllAddr_gfx11_gfx12_Impl<op, /*hasTFE=*/1, real_name> { 2555 defvar ps = get_BUF_ps<NAME>; 2556 if !ne(ps.Mnemonic, real_name) then 2557 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>; 2558} 2559 2560multiclass MUBUF_Real_Atomic_gfx11_impl<bits<8> op, bit is_return, 2561 string real_name> { 2562 defvar Rtn = !if(is_return, "_RTN", ""); 2563 defm _BOTHEN#Rtn : MUBUF_Real_gfx11<op, real_name>; 2564 defm _IDXEN#Rtn : MUBUF_Real_gfx11<op, real_name>; 2565 defm _OFFEN#Rtn : MUBUF_Real_gfx11<op, real_name>; 2566 defm _OFFSET#Rtn : MUBUF_Real_gfx11<op, real_name>; 2567} 2568 2569multiclass MUBUF_Real_Atomic_gfx12_impl<bits<8> op, bit is_return, 2570 string real_name = get_BUF_ps<NAME>.Mnemonic> { 2571 defvar Rtn = !if(is_return, "_RTN", ""); 2572 defm _VBUFFER_BOTHEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2573 defm _VBUFFER_IDXEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2574 defm _VBUFFER_OFFEN#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2575 defm _VBUFFER_OFFSET#Rtn : VBUFFER_MUBUF_Real_gfx12<op, real_name>; 2576} 2577 2578multiclass MUBUF_Real_Atomic_gfx11_gfx12_impl<bits<8> op, bit is_return, 2579 string real_name> : 2580 MUBUF_Real_Atomic_gfx11_impl<op, is_return, real_name>, 2581 MUBUF_Real_Atomic_gfx12_impl<op, is_return, real_name>; 2582 2583multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> : 2584 MUBUF_Real_Atomic_gfx12_impl<op, 0>, 2585 MUBUF_Real_Atomic_gfx12_impl<op, 1>; 2586 2587multiclass MUBUF_Real_Atomic_gfx11<bits<8> op, string real_name> : 2588 MUBUF_Real_Atomic_gfx11_impl<op, 0, real_name>, 2589 MUBUF_Real_Atomic_gfx11_impl<op, 1, real_name> { 2590 defvar ps = get_BUF_ps<NAME>; 2591 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>; 2592} 2593 2594multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op, 2595 string gfx12_name = get_BUF_ps<NAME>.Mnemonic, 2596 string gfx11_name = gfx12_name> : 2597 MUBUF_Real_Atomic_gfx11_impl<op, 0, gfx11_name>, 2598 MUBUF_Real_Atomic_gfx11_impl<op, 1, gfx11_name>, 2599 MUBUF_Real_Atomic_gfx12_impl<op, 0, gfx12_name>, 2600 MUBUF_Real_Atomic_gfx12_impl<op, 1, gfx12_name> { 2601 defvar ps = get_BUF_ps<NAME>; 2602 if !ne(ps.Mnemonic, gfx11_name) then 2603 def : Mnem_gfx11<ps.Mnemonic, gfx11_name>; 2604 if !ne(ps.Mnemonic, gfx12_name) then 2605 def : Mnem_gfx12<ps.Mnemonic, gfx12_name>; 2606 if !ne(gfx11_name, gfx12_name) then 2607 def : Mnem_gfx12<gfx11_name, gfx12_name>; 2608} 2609 2610defm BUFFER_GL0_INV : MUBUF_Real_gfx11<0x02B>; 2611defm BUFFER_GL1_INV : MUBUF_Real_gfx11<0x02C>; 2612 2613defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x014, "buffer_load_b32">; 2614defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x015, "buffer_load_b64">; 2615defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x016, "buffer_load_b96">; 2616defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x017, "buffer_load_b128">; 2617defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x020, "buffer_load_d16_b16">; 2618defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x008, "buffer_load_d16_format_x">; 2619defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x009, "buffer_load_d16_format_xy">; 2620defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00a, "buffer_load_d16_format_xyz">; 2621defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00b, "buffer_load_d16_format_xyzw">; 2622defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x023, "buffer_load_d16_hi_b16">; 2623defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x026, "buffer_load_d16_hi_format_x">; 2624defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x022, "buffer_load_d16_hi_i8">; 2625defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x021, "buffer_load_d16_hi_u8">; 2626defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01f, "buffer_load_d16_i8">; 2627defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01e, "buffer_load_d16_u8">; 2628defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2629defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2630defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2631defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2632defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x011, "buffer_load_i8">; 2633defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x013, "buffer_load_i16">; 2634defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x010, "buffer_load_u8">; 2635defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x012, "buffer_load_u16">; 2636defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>; 2637defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>; 2638defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>; 2639defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>; 2640defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>; 2641defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>; 2642defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_gfx12<0x018, "buffer_store_b8">; 2643defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_gfx12<0x019, "buffer_store_b16">; 2644defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12<0x01A, "buffer_store_b32">; 2645defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01B, "buffer_store_b64">; 2646defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01C, "buffer_store_b96">; 2647defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12<0x01D, "buffer_store_b128">; 2648defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x00C, "buffer_store_d16_format_x">; 2649defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x00D, "buffer_store_d16_format_xy">; 2650defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x00E, "buffer_store_d16_format_xyz">; 2651defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x00F, "buffer_store_d16_format_xyzw">; 2652defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x024, "buffer_store_d16_hi_b8">; 2653defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12<0x025, "buffer_store_d16_hi_b16">; 2654defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x027, "buffer_store_d16_hi_format_x">; 2655defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2656defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2657defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2658defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2659defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11_gfx12<0x056>; 2660defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_gfx12<0x035, "buffer_atomic_add_u32">; 2661defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x043, "buffer_atomic_add_u64">; 2662defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_gfx12<0x03C, "buffer_atomic_and_b32">; 2663defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x049, "buffer_atomic_and_b64">; 2664defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x034, "buffer_atomic_cmpswap_b32">; 2665defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x042, "buffer_atomic_cmpswap_b64">; 2666defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11<0x050, "buffer_atomic_cmpswap_f32">; 2667defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Real_Atomic_gfx12<0x050>; 2668defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">; 2669defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12<0x040, "buffer_atomic_dec_u32">; 2670defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04D, "buffer_atomic_dec_u64">; 2671defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12<0x03F, "buffer_atomic_inc_u32">; 2672defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04C, "buffer_atomic_inc_u64">; 2673defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">; 2674defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03A, "buffer_atomic_max_i32">; 2675defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x047, "buffer_atomic_max_i64">; 2676defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_gfx12<0x03B, "buffer_atomic_max_u32">; 2677defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x048, "buffer_atomic_max_u64">; 2678defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">; 2679defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x038, "buffer_atomic_min_i32">; 2680defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x045, "buffer_atomic_min_i64">; 2681defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_gfx12<0x039, "buffer_atomic_min_u32">; 2682defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x046, "buffer_atomic_min_u64">; 2683defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_gfx12<0x03D, "buffer_atomic_or_b32">; 2684defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04A, "buffer_atomic_or_b64">; 2685defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_gfx12<0x036, "buffer_atomic_sub_u32">; 2686defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x044, "buffer_atomic_sub_u64">; 2687defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_gfx12<0x033, "buffer_atomic_swap_b32">; 2688defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x041, "buffer_atomic_swap_b64">; 2689defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_gfx12<0x03E, "buffer_atomic_xor_b32">; 2690defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_gfx12<0x04B, "buffer_atomic_xor_b64">; 2691defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_gfx12<0x059>; 2692defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_gfx12<0x05a>; 2693 2694//===----------------------------------------------------------------------===// 2695// MUBUF - GFX10. 2696//===----------------------------------------------------------------------===// 2697 2698multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> { 2699 defm _BOTHEN : MUBUF_Real_gfx10<op>; 2700 defm _IDXEN : MUBUF_Real_gfx10<op>; 2701 defm _OFFEN : MUBUF_Real_gfx10<op>; 2702 defm _OFFSET : MUBUF_Real_gfx10<op>; 2703} 2704multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2705 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2706 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2707} 2708multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> { 2709 defm _OFFSET : MUBUF_Real_gfx10<op>; 2710 defm _OFFEN : MUBUF_Real_gfx10<op>; 2711 defm _IDXEN : MUBUF_Real_gfx10<op>; 2712 defm _BOTHEN : MUBUF_Real_gfx10<op>; 2713 2714 if !not(isTFE) then { 2715 defm _LDS_OFFSET : MUBUF_Real_gfx10<op>; 2716 defm _LDS_OFFEN : MUBUF_Real_gfx10<op>; 2717 defm _LDS_IDXEN : MUBUF_Real_gfx10<op>; 2718 defm _LDS_BOTHEN : MUBUF_Real_gfx10<op>; 2719 } 2720} 2721multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op, string psName = NAME, 2722 string asmName = !cast<MUBUF_Pseudo>(psName).Mnemonic> { 2723 defm _BOTHEN_RTN : MUBUF_Real_gfx10<op, psName#"_BOTHEN_RTN", asmName>; 2724 defm _IDXEN_RTN : MUBUF_Real_gfx10<op, psName#"_IDXEN_RTN", asmName>; 2725 defm _OFFEN_RTN : MUBUF_Real_gfx10<op, psName#"_OFFEN_RTN", asmName>; 2726 defm _OFFSET_RTN : MUBUF_Real_gfx10<op, psName#"_OFFSET_RTN", asmName>; 2727} 2728multiclass MUBUF_Real_Atomics_gfx10<bits<8> op, string psName = NAME, 2729 string asmName = get_BUF_ps<psName>.Mnemonic> : 2730 MUBUF_Real_Atomics_RTN_gfx10<op, psName, asmName> { 2731 defm _BOTHEN : MUBUF_Real_gfx10<op, psName#"_BOTHEN", asmName>; 2732 defm _IDXEN : MUBUF_Real_gfx10<op, psName#"_IDXEN", asmName>; 2733 defm _OFFEN : MUBUF_Real_gfx10<op, psName#"_OFFEN", asmName>; 2734 defm _OFFSET : MUBUF_Real_gfx10<op, psName#"_OFFSET", asmName>; 2735} 2736 2737defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2738defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2739defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2740defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2741defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2742defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2743defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2744defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2745defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2746defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2747defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2748defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2749defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2750defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2751defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2752defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2753defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2754defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2755 2756defm BUFFER_GL0_INV : MUBUF_Real_gfx10<0x071>; 2757defm BUFFER_GL1_INV : MUBUF_Real_gfx10<0x072>; 2758 2759//===----------------------------------------------------------------------===// 2760// MUBUF - GFX6, GFX7, GFX10. 2761//===----------------------------------------------------------------------===// 2762 2763multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> { 2764 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>; 2765 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>; 2766 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>; 2767 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>; 2768 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>; 2769} 2770multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2771 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2772 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2773} 2774multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> { 2775 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op>; 2776 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op>; 2777 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op>; 2778 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op>; 2779 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op>; 2780 2781 if !not(isTFE) then { 2782 defm _LDS_OFFSET : MUBUF_Real_gfx6_gfx7<op>; 2783 defm _LDS_ADDR64 : MUBUF_Real_gfx6_gfx7<op>; 2784 defm _LDS_OFFEN : MUBUF_Real_gfx6_gfx7<op>; 2785 defm _LDS_IDXEN : MUBUF_Real_gfx6_gfx7<op>; 2786 defm _LDS_BOTHEN : MUBUF_Real_gfx6_gfx7<op>; 2787 } 2788} 2789multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op, string psName, string asmName> { 2790 defm _ADDR64 : MUBUF_Real_gfx6_gfx7<op, psName#"_ADDR64", asmName>; 2791 defm _BOTHEN : MUBUF_Real_gfx6_gfx7<op, psName#"_BOTHEN", asmName>; 2792 defm _IDXEN : MUBUF_Real_gfx6_gfx7<op, psName#"_IDXEN", asmName>; 2793 defm _OFFEN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFEN", asmName>; 2794 defm _OFFSET : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFSET", asmName>; 2795 2796 defm _ADDR64_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_ADDR64_RTN", asmName>; 2797 defm _BOTHEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_BOTHEN_RTN", asmName>; 2798 defm _IDXEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_IDXEN_RTN", asmName>; 2799 defm _OFFEN_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFEN_RTN", asmName>; 2800 defm _OFFSET_RTN : MUBUF_Real_gfx6_gfx7<op, psName#"_OFFSET_RTN", asmName>; 2801} 2802 2803multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2804 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2805 2806multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> : 2807 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>, 2808 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>; 2809 2810multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> { 2811 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>; 2812 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>; 2813} 2814 2815multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op, string psName = NAME, 2816 string asmName = get_BUF_ps<psName>.Mnemonic> : 2817 MUBUF_Real_Atomics_gfx6_gfx7<op, psName, asmName>, 2818 MUBUF_Real_Atomics_gfx10<op, psName, asmName>; 2819 2820// FIXME-GFX6: Following instructions are available only on GFX6. 2821//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2822//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2823 2824defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2825defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2826defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2827defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2828defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2829defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2830defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2831defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2832defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2833defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2834defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2835defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2836defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2837defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2838defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2839defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2840defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2841defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2842defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2843defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2844defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2845defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2846 2847defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2848defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2849defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2850defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2851defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2852defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2853defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2854defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2855defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2856defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2857defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2858defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2859defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2860defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2861defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2862defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2863defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2864defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2865defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2866defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2867defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2868defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2869defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2870defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2871defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2872defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2873defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2874defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2875defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2876// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2877defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2878defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f, "BUFFER_ATOMIC_MIN_F64", "buffer_atomic_fmin_x2">; 2879defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060, "BUFFER_ATOMIC_MAX_F64", "buffer_atomic_fmax_x2">; 2880 2881defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>; 2882 2883defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2884defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2885defm BUFFER_WBINVL1 : MUBUF_Real_gfx6_gfx7<0x071>; 2886 2887//===----------------------------------------------------------------------===// 2888// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2889//===----------------------------------------------------------------------===// 2890 2891class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2892 string real_name = ps.Mnemonic> : 2893 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2894 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2895 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2896 let Inst{31-26} = 0x3a; //encoding 2897 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2898 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2899 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2900 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2901} 2902 2903multiclass MTBUF_Real_gfx11<bits<4> op, string real_name> { 2904 defvar ps = !cast<MTBUF_Pseudo>(NAME); 2905 def _gfx11 : Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2906 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2907 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2908 let Inst{18-15} = op; 2909 let Inst{25-19} = format; 2910 let Inst{53} = ps.tfe; 2911 let Inst{54} = ps.offen; 2912 let Inst{55} = ps.idxen; 2913 let AssemblerPredicate = isGFX11Only; 2914 let DecoderNamespace = "GFX11"; 2915 } 2916} 2917 2918class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2919 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2920 let Inst{12} = ps.offen; 2921 let Inst{13} = ps.idxen; 2922 let Inst{18-16} = op; 2923 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2924 let Inst{55} = ps.tfe; 2925} 2926 2927//===----------------------------------------------------------------------===// 2928// MTBUF - GFX11. 2929//===----------------------------------------------------------------------===// 2930 2931multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, string real_name> { 2932 defm _BOTHEN : MTBUF_Real_gfx11<op, real_name>; 2933 defm _IDXEN : MTBUF_Real_gfx11<op, real_name>; 2934 defm _OFFEN : MTBUF_Real_gfx11<op, real_name>; 2935 defm _OFFSET : MTBUF_Real_gfx11<op, real_name>; 2936 2937 defm _VBUFFER_BOTHEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2938 defm _VBUFFER_IDXEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2939 defm _VBUFFER_OFFEN : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2940 defm _VBUFFER_OFFSET : VBUFFER_MTBUF_Real_gfx12<op, real_name>; 2941} 2942 2943multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op, 2944 string real_name = get_BUF_ps<NAME>.Mnemonic> 2945 : MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, real_name> { 2946 defvar ps = get_BUF_ps<NAME>; 2947 if !ne(ps.Mnemonic, real_name) then 2948 def : Mnem_gfx11_gfx12<ps.Mnemonic, real_name>; 2949} 2950 2951defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x008, "tbuffer_load_d16_format_x">; 2952defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x009, "tbuffer_load_d16_format_xy">; 2953defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00a, "tbuffer_load_d16_format_xyz">; 2954defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00b, "tbuffer_load_d16_format_xyzw">; 2955defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2956defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2957defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2958defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2959defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x00c, "tbuffer_store_d16_format_x">; 2960defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x00d, "tbuffer_store_d16_format_xy">; 2961defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x00e, "tbuffer_store_d16_format_xyz">; 2962defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x00f, "tbuffer_store_d16_format_xyzw">; 2963defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2964defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2965defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2966defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2967 2968//===----------------------------------------------------------------------===// 2969// MTBUF - GFX10. 2970//===----------------------------------------------------------------------===// 2971 2972multiclass MTBUF_Real_gfx10<bits<4> op> { 2973 defvar ps = !cast<MTBUF_Pseudo>(NAME); 2974 def _gfx10 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2975 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2976 let Inst{25-19} = format; 2977 let Inst{53} = op{3}; 2978 let AssemblerPredicate = isGFX10Only; 2979 let DecoderNamespace = "GFX10"; 2980 } 2981} 2982 2983multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2984 defm _BOTHEN : MTBUF_Real_gfx10<op>; 2985 defm _IDXEN : MTBUF_Real_gfx10<op>; 2986 defm _OFFEN : MTBUF_Real_gfx10<op>; 2987 defm _OFFSET : MTBUF_Real_gfx10<op>; 2988} 2989 2990defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2991defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2992defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2993defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2994defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2995defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2996defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2997defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2998 2999//===----------------------------------------------------------------------===// 3000// MTBUF - GFX6, GFX7, GFX10. 3001//===----------------------------------------------------------------------===// 3002 3003multiclass MTBUF_Real_gfx6_gfx7<bits<4> op> { 3004 defvar ps = !cast<MTBUF_Pseudo>(NAME); 3005 def _gfx6_gfx7 : Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 3006 let Inst{15} = ps.addr64; 3007 let Inst{22-19} = dfmt; 3008 let Inst{25-23} = nfmt; 3009 let AssemblerPredicate = isGFX6GFX7; 3010 let DecoderNamespace = "GFX6GFX7"; 3011 } 3012} 3013 3014multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 3015 defm _ADDR64 : MTBUF_Real_gfx6_gfx7<op>; 3016 defm _BOTHEN : MTBUF_Real_gfx6_gfx7<op>; 3017 defm _IDXEN : MTBUF_Real_gfx6_gfx7<op>; 3018 defm _OFFEN : MTBUF_Real_gfx6_gfx7<op>; 3019 defm _OFFSET : MTBUF_Real_gfx6_gfx7<op>; 3020} 3021 3022multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 3023 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 3024 3025defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 3026defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 3027defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 3028defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 3029defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 3030defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 3031defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 3032defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 3033 3034//===----------------------------------------------------------------------===// 3035// GFX8, GFX9 (VI). 3036//===----------------------------------------------------------------------===// 3037 3038class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 3039 bit has_sccb = ps.has_sccb> : 3040 MUBUF_Real<ps>, 3041 Enc64, 3042 SIMCInstr<ps.PseudoInstr, Enc> { 3043 3044 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3045 let Inst{12} = ps.offen; 3046 let Inst{13} = ps.idxen; 3047 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3048 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3049 let Inst{16} = ps.lds; 3050 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3051 let Inst{24-18} = op; 3052 let Inst{31-26} = 0x38; //encoding 3053 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3054 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3055 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3056 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3057} 3058 3059multiclass MUBUF_Real_vi <bits<7> op, 3060 bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> { 3061 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3062 def _vi : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 3063 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3064 let DecoderNamespace = "GFX8"; 3065 3066 let Inst{55} = ps.tfe; 3067 } 3068} 3069 3070multiclass MUBUF_Real_gfx90a <bits<7> op, 3071 bit has_sccb = !cast<MUBUF_Pseudo>(NAME).has_sccb> { 3072 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3073 def _gfx90a : MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 3074 let AssemblerPredicate = isGFX90APlus; 3075 let DecoderNamespace = "GFX90A"; 3076 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 3077 ps.AsmOperands); 3078 3079 let Inst{55} = acc; 3080 } 3081} 3082 3083class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 3084 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 3085 let AssemblerPredicate = isGFX940Plus; 3086 let DecoderNamespace = "GFX9"; 3087 let AsmString = ps.Mnemonic # ps.AsmOperands; 3088 3089 let Inst{55} = acc; 3090} 3091 3092multiclass MUBUF_Real_vi_gfx90a<bits<7> op, bit isTFE = 0> : MUBUF_Real_vi<op> { 3093 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3094 3095 if !not(isTFE) then { 3096 if !not(ps.FPAtomic) then 3097 defm NAME : MUBUF_Real_gfx90a<op>; 3098 } 3099 3100 if ps.FPAtomic then { 3101 let AssemblerPredicate = isGFX90AOnly in 3102 defm NAME : MUBUF_Real_gfx90a<op, 0>; 3103 3104 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 3105 } 3106} 3107 3108multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> { 3109 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, isTFE>; 3110 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, isTFE>; 3111 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, isTFE>; 3112 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, isTFE>; 3113} 3114 3115multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> { 3116 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>; 3117 if hasTFE then 3118 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>; 3119} 3120 3121multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> { 3122 defm _OFFSET : MUBUF_Real_vi <op>; 3123 defm _OFFEN : MUBUF_Real_vi <op>; 3124 defm _IDXEN : MUBUF_Real_vi <op>; 3125 defm _BOTHEN : MUBUF_Real_vi <op>; 3126 3127 if !not(isTFE) then { 3128 defm _LDS_OFFSET : MUBUF_Real_vi <op>; 3129 defm _LDS_OFFEN : MUBUF_Real_vi <op>; 3130 defm _LDS_IDXEN : MUBUF_Real_vi <op>; 3131 defm _LDS_BOTHEN : MUBUF_Real_vi <op>; 3132 3133 defm _OFFSET : MUBUF_Real_gfx90a <op>; 3134 defm _OFFEN : MUBUF_Real_gfx90a <op>; 3135 defm _IDXEN : MUBUF_Real_gfx90a <op>; 3136 defm _BOTHEN : MUBUF_Real_gfx90a <op>; 3137 3138 defm _LDS_OFFSET : MUBUF_Real_gfx90a <op>; 3139 defm _LDS_OFFEN : MUBUF_Real_gfx90a <op>; 3140 defm _LDS_IDXEN : MUBUF_Real_gfx90a <op>; 3141 defm _LDS_BOTHEN : MUBUF_Real_gfx90a <op>; 3142 } 3143} 3144 3145multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 3146 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>; 3147 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>; 3148} 3149 3150multiclass MUBUF_Real_gfx80 <bits<7> op> { 3151 defvar ps = !cast<MUBUF_Pseudo>(NAME); 3152 def _gfx80 : MUBUF_Real<ps>, 3153 Enc64, 3154 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3155 let AssemblerPredicate=HasUnpackedD16VMem; 3156 let DecoderNamespace="GFX80_UNPACKED"; 3157 3158 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3159 let Inst{12} = ps.offen; 3160 let Inst{13} = ps.idxen; 3161 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3162 let Inst{16} = ps.lds; 3163 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3164 let Inst{24-18} = op; 3165 let Inst{31-26} = 0x38; //encoding 3166 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3167 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3168 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3169 let Inst{55} = ps.tfe; 3170 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3171 } 3172} 3173 3174multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> { 3175 defm _OFFSET : MUBUF_Real_gfx80 <op>; 3176 defm _OFFEN : MUBUF_Real_gfx80 <op>; 3177 defm _IDXEN : MUBUF_Real_gfx80 <op>; 3178 defm _BOTHEN : MUBUF_Real_gfx80 <op>; 3179} 3180 3181multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 3182 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3183 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3184} 3185 3186multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 3187 MUBUF_Real_AllAddr_vi<op, 0> { 3188 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op>; 3189 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op>; 3190 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op>; 3191 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op>; 3192} 3193 3194defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 3195defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 3196defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 3197defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 3198defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 3199defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 3200defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 3201defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 3202let SubtargetPredicate = HasUnpackedD16VMem in { 3203 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 3204 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 3205 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 3206 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 3207 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 3208 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 3209 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 3210 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 3211} // End HasUnpackedD16VMem. 3212let SubtargetPredicate = HasPackedD16VMem in { 3213 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 3214 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 3215 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 3216 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 3217 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 3218 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 3219 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 3220 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 3221} // End HasPackedD16VMem. 3222defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 3223defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 3224defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 3225defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 3226defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 3227defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 3228defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 3229defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 3230defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 3231defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 3232defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 3233defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 3234defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 3235defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 3236defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 3237defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 3238 3239defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 3240defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 3241defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 3242defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 3243defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 3244defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 3245 3246defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 3247defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 3248 3249defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 3250defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 3251defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 3252defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 3253defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 3254defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 3255defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 3256defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 3257defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 3258defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 3259defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 3260defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 3261defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 3262 3263defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 3264defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 3265defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 3266defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 3267defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 3268defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 3269defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 3270defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 3271defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 3272defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 3273defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 3274defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 3275defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 3276 3277defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d>; 3278 3279let AssemblerPredicate = isGFX8GFX9 in { 3280defm BUFFER_WBINVL1 : MUBUF_Real_vi <0x3e>; 3281defm BUFFER_WBINVL1_VOL : MUBUF_Real_vi <0x3f>; 3282} // End AssemblerPredicate = isGFX8GFX9 3283 3284 3285defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 3286defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 3287 3288let SubtargetPredicate = isGFX90APlus in { 3289 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 3290 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 3291 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 3292} // End SubtargetPredicate = isGFX90APlus 3293 3294let AsmString = BUFFER_WBL2.Mnemonic, // drop flags 3295 AssemblerPredicate = isGFX90AOnly, 3296 SubtargetPredicate = isGFX90AOnly in 3297defm BUFFER_WBL2 : MUBUF_Real_gfx90a<0x28>; 3298defm BUFFER_INVL2 : MUBUF_Real_gfx90a<0x29>; 3299 3300let SubtargetPredicate = isGFX940Plus in { 3301def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 3302def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 3303} 3304 3305class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 3306 MTBUF_Real<ps>, 3307 Enc64, 3308 SIMCInstr<ps.PseudoInstr, Enc> { 3309 3310 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3311 let Inst{12} = ps.offen; 3312 let Inst{13} = ps.idxen; 3313 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3314 let Inst{18-15} = op; 3315 let Inst{22-19} = dfmt; 3316 let Inst{25-23} = nfmt; 3317 let Inst{31-26} = 0x3a; //encoding 3318 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3319 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3320 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3321 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3322 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3323 let Inst{55} = ps.tfe; 3324 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3325} 3326 3327class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 3328 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 3329 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3330 let DecoderNamespace = "GFX8"; 3331 3332 let Inst{55} = ps.tfe; 3333} 3334 3335class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 3336 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 3337 let AssemblerPredicate = isGFX90APlus; 3338 let DecoderNamespace = "GFX90A"; 3339 let AsmString = ps.Mnemonic # ps.AsmOperands; 3340 3341 let Inst{55} = acc; 3342} 3343 3344multiclass MTBUF_Real_vi_gfx90a<bits<4> op> { 3345 defvar ps = !cast<MTBUF_Pseudo>(NAME); 3346 def _vi : MTBUF_Real_vi<op, ps>; 3347 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 3348} 3349 3350multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 3351 defm _OFFSET : MTBUF_Real_vi_gfx90a <op>; 3352 defm _OFFEN : MTBUF_Real_vi_gfx90a <op>; 3353 defm _IDXEN : MTBUF_Real_vi_gfx90a <op>; 3354 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op>; 3355} 3356 3357multiclass MTBUF_Real_gfx80 <bits<4> op> { 3358 defvar ps = !cast<MTBUF_Pseudo>(NAME); 3359 def _gfx80 : MTBUF_Real<ps>, 3360 Enc64, 3361 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3362 let AssemblerPredicate=HasUnpackedD16VMem; 3363 let DecoderNamespace="GFX80_UNPACKED"; 3364 3365 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3366 let Inst{12} = ps.offen; 3367 let Inst{13} = ps.idxen; 3368 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3369 let Inst{18-15} = op; 3370 let Inst{22-19} = dfmt; 3371 let Inst{25-23} = nfmt; 3372 let Inst{31-26} = 0x3a; //encoding 3373 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3374 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3375 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3376 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3377 let Inst{55} = ps.tfe; 3378 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3379 } 3380} 3381 3382multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 3383 defm _OFFSET : MTBUF_Real_gfx80 <op>; 3384 defm _OFFEN : MTBUF_Real_gfx80 <op>; 3385 defm _IDXEN : MTBUF_Real_gfx80 <op>; 3386 defm _BOTHEN : MTBUF_Real_gfx80 <op>; 3387} 3388 3389defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 3390defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 3391defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 3392defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 3393defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 3394defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 3395defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 3396defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 3397let SubtargetPredicate = HasUnpackedD16VMem in { 3398 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 3399 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 3400 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 3401 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 3402 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 3403 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 3404 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 3405 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 3406} // End HasUnpackedD16VMem. 3407let SubtargetPredicate = HasPackedD16VMem in { 3408 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 3409 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 3410 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 3411 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 3412 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 3413 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 3414 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 3415 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 3416} // End HasUnpackedD16VMem. 3417 3418def MUBUFInfoTable : GenericTable { 3419 let FilterClass = "MUBUF_Pseudo"; 3420 let CppTypeName = "MUBUFInfo"; 3421 let Fields = [ 3422 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3423 "IsBufferInv", "tfe" 3424 ]; 3425 3426 let PrimaryKey = ["Opcode"]; 3427 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3428} 3429 3430def getMUBUFInfoFromOpcode : SearchIndex { 3431 let Table = MUBUFInfoTable; 3432 let Key = ["Opcode"]; 3433} 3434 3435def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3436 let Table = MUBUFInfoTable; 3437 let Key = ["BaseOpcode", "elements"]; 3438} 3439 3440def MTBUFInfoTable : GenericTable { 3441 let FilterClass = "MTBUF_Pseudo"; 3442 let CppTypeName = "MTBUFInfo"; 3443 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3444 3445 let PrimaryKey = ["Opcode"]; 3446 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3447} 3448 3449def getMTBUFInfoFromOpcode : SearchIndex { 3450 let Table = MTBUFInfoTable; 3451 let Key = ["Opcode"]; 3452} 3453 3454def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3455 let Table = MTBUFInfoTable; 3456 let Key = ["BaseOpcode", "elements"]; 3457} 3458