1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFSOffset : ComplexPattern<iPTR, 1, "SelectBUFSOffset">; 16 17def BUFAddrKind { 18 int Offset = 0; 19 int OffEn = 1; 20 int IdxEn = 2; 21 int BothEn = 3; 22 int Addr64 = 4; 23} 24 25class getAddrName<int addrKind> { 26 string ret = 27 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 28 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 29 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 30 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 31 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 32 ""))))); 33} 34 35class MUBUFAddr64Table <bit is_addr64, string Name> { 36 bit IsAddr64 = is_addr64; 37 string OpName = Name; 38} 39 40class MTBUFAddr64Table <bit is_addr64, string Name> { 41 bit IsAddr64 = is_addr64; 42 string OpName = Name; 43} 44 45 46//===----------------------------------------------------------------------===// 47// BUF class (base class for MTBUF and MUBUF pseudos) 48//===----------------------------------------------------------------------===// 49 50class BUF_Pseudo <string opName, dag outs, dag ins, 51 string asmOps, list<dag> pattern=[]> : 52 InstSI<outs, ins, "", pattern>, 53 SIMCInstr<opName, SIEncodingFamily.NONE> { 54 55 let isPseudo = 1; 56 let isCodeGenOnly = 1; 57 let Size = 8; 58 let UseNamedOperandTable = 1; 59 60 string Mnemonic = opName; 61 string AsmOperands = asmOps; 62 63 Instruction Opcode = !cast<Instruction>(NAME); 64 65 66 let VM_CNT = 1; 67 let EXP_CNT = 1; 68 69 let Uses = [EXEC]; 70 let hasSideEffects = 0; 71 let SchedRW = [WriteVMEM]; 72 73 74 75 bits<1> offen = 0; 76 bits<1> idxen = 0; 77 bits<1> addr64 = 0; 78 bits<1> lds = 0; 79 bits<1> has_vdata = !not(lds); 80 bits<1> has_vaddr = 1; 81 bits<1> has_glc = 1; 82 bits<1> has_dlc = 1; 83 bits<1> glc_value = 0; // the value for glc if no such operand 84 bits<1> dlc_value = 0; // the value for dlc if no such operand 85 bits<1> has_srsrc = 1; 86 bits<1> has_soffset = 1; 87 bits<1> has_offset = 1; 88 bits<1> has_slc = 1; 89 bits<1> tfe = ?; 90 bits<4> elements = 0; 91 bits<1> has_sccb = 1; 92 bits<1> sccb_value = 0; 93 bits<1> IsBufferInv = 0; 94} 95 96 97 98//===----------------------------------------------------------------------===// 99// MTBUF classes 100//===----------------------------------------------------------------------===// 101 102class MTBUFGetBaseOpcode<string Op> { 103 string ret = !subst("FORMAT_XY", "FORMAT_X", 104 !subst("FORMAT_XYZ", "FORMAT_X", 105 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 106} 107 108 109class MTBUF_Pseudo <string opName, dag outs, dag ins, 110 string asmOps, list<dag> pattern=[]> : 111 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 112 113 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 114 let MTBUF = 1; 115} 116 117class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 118 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 119 120 let isPseudo = 0; 121 let isCodeGenOnly = 0; 122 123 let VM_CNT = 1; 124 let EXP_CNT = 1; 125 let MTBUF = 1; 126 127 // copy relevant pseudo op flags 128 let UseNamedOperandTable = ps.UseNamedOperandTable; 129 let SubtargetPredicate = ps.SubtargetPredicate; 130 let AsmMatchConverter = ps.AsmMatchConverter; 131 let Constraints = ps.Constraints; 132 let DisableEncoding = ps.DisableEncoding; 133 let TSFlags = ps.TSFlags; 134 let SchedRW = ps.SchedRW; 135 let mayLoad = ps.mayLoad; 136 let mayStore = ps.mayStore; 137 let IsAtomicRet = ps.IsAtomicRet; 138 let IsAtomicNoRet = ps.IsAtomicNoRet; 139 140 bits<12> offset; 141 bits<5> cpol; 142 bits<7> format; 143 bits<8> vaddr; 144 bits<10> vdata; 145 bits<7> srsrc; 146 bits<8> soffset; 147 148 bits<4> dfmt = format{3-0}; 149 bits<3> nfmt = format{6-4}; 150 151 // GFX90A+ only: instruction uses AccVGPR for data 152 // Bit supersedes tfe. 153 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 154} 155 156class getMTBUFInsDA<list<RegisterClass> vdataList, 157 list<RegisterClass> vaddrList=[], bit hasGFX12Enc> { 158 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 159 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 160 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 161 162 dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), 163 (ins SCSrc_b32:$soffset)); 164 165 dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, 166 (ins offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz)); 167 168 dag Inputs = !if(!empty(vaddrList), 169 NonVaddrInputs, 170 !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 171 dag ret = !if(!empty(vdataList), 172 Inputs, 173 !con((ins vdata_op:$vdata), Inputs)); 174} 175 176class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit hasGFX12Enc> { 177 dag ret = 178 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasGFX12Enc>.ret, 179 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32], hasGFX12Enc>.ret, 180 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32], hasGFX12Enc>.ret, 181 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasGFX12Enc>.ret, 182 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasGFX12Enc>.ret, 183 (ins)))))); 184} 185 186class getMTBUFAsmOps<int addrKind> { 187 string Pfx = 188 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 189 !if(!eq(addrKind, BUFAddrKind.OffEn), 190 "$vaddr, $srsrc,$format $soffset offen", 191 !if(!eq(addrKind, BUFAddrKind.IdxEn), 192 "$vaddr, $srsrc,$format $soffset idxen", 193 !if(!eq(addrKind, BUFAddrKind.BothEn), 194 "$vaddr, $srsrc,$format $soffset idxen offen", 195 !if(!eq(addrKind, BUFAddrKind.Addr64), 196 "$vaddr, $srsrc,$format $soffset addr64", 197 ""))))); 198 string ret = " $vdata, " # Pfx # "$offset$cpol"; 199} 200 201class MTBUF_SetupAddr<int addrKind> { 202 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 203 !eq(addrKind, BUFAddrKind.BothEn)); 204 205 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 206 !eq(addrKind, BUFAddrKind.BothEn)); 207 208 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 209 210 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 211} 212 213class MTBUF_Load_Pseudo <string opName, 214 int addrKind, 215 RegisterClass vdataClass, 216 int elems, 217 bit hasGFX12Enc = 0, 218 list<dag> pattern=[], 219 // Workaround bug bz30254 220 int addrKindCopy = addrKind> 221 : MTBUF_Pseudo<opName, 222 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 223 getMTBUFIns<addrKindCopy, [], hasGFX12Enc>.ret, 224 getMTBUFAsmOps<addrKindCopy>.ret, 225 pattern>, 226 MTBUF_SetupAddr<addrKindCopy> { 227 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 228 let mayLoad = 1; 229 let mayStore = 0; 230 let elements = elems; 231} 232 233multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterClass vdataClass, 234 int elems, bit hasGFX12Enc> { 235 236 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>, 237 MTBUFAddr64Table<0, NAME>; 238 239 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasGFX12Enc>, 240 MTBUFAddr64Table<1, NAME>; 241 242 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 243 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 244 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 245 246 let DisableWQM = 1 in { 247 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>; 248 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 249 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 250 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 251 } 252} 253 254multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 255 int elems> { 256 defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>; 257 defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>; 258} 259 260class MTBUF_Store_Pseudo <string opName, 261 int addrKind, 262 RegisterClass vdataClass, 263 int elems, 264 bit hasGFX12Enc = 0, 265 list<dag> pattern=[], 266 // Workaround bug bz30254 267 int addrKindCopy = addrKind, 268 RegisterClass vdataClassCopy = vdataClass> 269 : MTBUF_Pseudo<opName, 270 (outs), 271 getMTBUFIns<addrKindCopy, [vdataClassCopy], hasGFX12Enc>.ret, 272 getMTBUFAsmOps<addrKindCopy>.ret, 273 pattern>, 274 MTBUF_SetupAddr<addrKindCopy> { 275 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 276 let mayLoad = 0; 277 let mayStore = 1; 278 let elements = elems; 279} 280 281multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterClass vdataClass, 282 int elems, bit hasGFX12Enc> { 283 284 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>, 285 MTBUFAddr64Table<0, NAME>; 286 287 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasGFX12Enc>, 288 MTBUFAddr64Table<1, NAME>; 289 290 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 291 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 292 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 293 294 let DisableWQM = 1 in { 295 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>; 296 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 297 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 298 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 299 } 300} 301 302multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 303 int elems> { 304 defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>; 305 defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>; 306} 307 308//===----------------------------------------------------------------------===// 309// MUBUF classes 310//===----------------------------------------------------------------------===// 311 312class MUBUFGetBaseOpcode<string Op> { 313 string ret = !subst("DWORDX2", "DWORD", 314 !subst("DWORDX3", "DWORD", 315 !subst("DWORDX4", "DWORD", Op))); 316} 317 318class MUBUF_Pseudo <string opName, dag outs, dag ins, 319 string asmOps, list<dag> pattern=[]> : 320 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 321 322 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 323 let MUBUF = 1; 324 let AsmMatchConverter = "cvtMubuf"; 325} 326 327class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 328 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 329 330 let isPseudo = 0; 331 let isCodeGenOnly = 0; 332 333 let VM_CNT = 1; 334 let EXP_CNT = 1; 335 let MUBUF = 1; 336 337 // copy relevant pseudo op flags 338 let SubtargetPredicate = ps.SubtargetPredicate; 339 let AsmMatchConverter = ps.AsmMatchConverter; 340 let OtherPredicates = ps.OtherPredicates; 341 let Constraints = ps.Constraints; 342 let DisableEncoding = ps.DisableEncoding; 343 let TSFlags = ps.TSFlags; 344 let UseNamedOperandTable = ps.UseNamedOperandTable; 345 let SchedRW = ps.SchedRW; 346 let mayLoad = ps.mayLoad; 347 let mayStore = ps.mayStore; 348 let IsAtomicRet = ps.IsAtomicRet; 349 let IsAtomicNoRet = ps.IsAtomicNoRet; 350 let VALU = ps.VALU; 351 let LGKM_CNT = ps.LGKM_CNT; 352 353 bits<12> offset; 354 bits<5> cpol; 355 bits<8> vaddr; 356 bits<10> vdata; 357 bits<7> srsrc; 358 bits<8> soffset; 359 360 // GFX90A+ only: instruction uses AccVGPR for data 361 // Bit supersedes tfe. 362 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 363} 364 365// For cache invalidation instructions. 366class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 367 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 368 369 let AsmMatchConverter = ""; 370 371 let hasSideEffects = 1; 372 let mayLoad = 0; 373 let mayStore = 0; 374 375 let IsBufferInv = 1; 376 // Set everything else to 0. 377 let offen = 0; 378 let idxen = 0; 379 let addr64 = 0; 380 let has_vdata = 0; 381 let has_vaddr = 0; 382 let has_glc = 0; 383 let has_dlc = 0; 384 let glc_value = 0; 385 let dlc_value = 0; 386 let has_srsrc = 0; 387 let has_soffset = 0; 388 let has_offset = 0; 389 let has_slc = 0; 390 let has_sccb = 0; 391 let sccb_value = 0; 392} 393 394class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> { 395 RegisterOperand tfeVDataOp = 396 !if(!eq(RC.Size, 32), AVLdSt_64, 397 !if(!eq(RC.Size, 64), AVLdSt_96, 398 !if(!eq(RC.Size, 96), AVLdSt_128, 399 !if(!eq(RC.Size, 128), AVLdSt_160, 400 RegisterOperand<VReg_1> // Invalid register. 401 )))); 402 403 RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret); 404} 405 406class getMUBUFInsDA<list<RegisterClass> vdataList, 407 list<RegisterClass> vaddrList, bit isTFE, bit hasGFX12Enc> { 408 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 409 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 410 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret; 411 412 dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 413 dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, (ins offset:$offset, CPol_0:$cpol, i1imm_0:$swz)); 414 415 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 416 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 417} 418 419class getMUBUFElements<ValueType vt> { 420 int ret = 421 !if(!eq(vt, f16), 1, 422 !if(!eq(vt, v2f16), 2, 423 !if(!eq(vt, v3f16), 3, 424 !if(!eq(vt, v4f16), 4, 425 !if(!eq(vt.Size, 32), 1, 426 !if(!eq(vt.Size, 64), 2, 427 !if(!eq(vt.Size, 96), 3, 428 !if(!eq(vt.Size, 128), 4, 0) 429 ) 430 ) 431 ) 432 ) 433 ) 434 ) 435 ); 436} 437 438class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE, bit hasGFX12Enc> { 439 dag ret = 440 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasGFX12Enc>.ret, 441 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasGFX12Enc>.ret, 442 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasGFX12Enc>.ret, 443 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasGFX12Enc>.ret, 444 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasGFX12Enc>.ret, 445 (ins)))))); 446} 447 448class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> { 449 string Vdata = !if(noVdata, " ", " $vdata, "); 450 string Lds = !if(isLds, " lds", ""); 451 string TFE = !if(isTFE, " tfe", ""); 452 string MainArgs = 453 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 454 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 455 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 456 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 457 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 458 ""))))); 459 string Offset = "$offset"; 460 string OtherArgs = "$cpol"; 461 462 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE; 463} 464 465class MUBUF_SetupAddr<int addrKind> { 466 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 467 !eq(addrKind, BUFAddrKind.BothEn)); 468 469 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 470 !eq(addrKind, BUFAddrKind.BothEn)); 471 472 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 473 474 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 475} 476 477class MUBUF_Load_Pseudo <string opName, 478 int addrKind, 479 ValueType vdata_vt, 480 bit HasTiedDest = 0, 481 bit isLds = 0, 482 bit isLdsOpc = 0, 483 bit isTFE = 0, 484 bit hasGFX12Enc = 0, 485 list<dag> pattern=[], 486 // Workaround bug bz30254 487 int addrKindCopy = addrKind, 488 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret.RegClass, 489 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret> 490 : MUBUF_Pseudo<opName, 491 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 492 !con(getMUBUFIns<addrKindCopy, [], isTFE, hasGFX12Enc>.ret, 493 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 494 getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret, 495 pattern>, 496 MUBUF_SetupAddr<addrKindCopy> { 497 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") # 498 "_" # getAddrName<addrKindCopy>.ret; 499 let AsmMatchConverter = "cvtMubuf"; 500 501 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 502 let LGKM_CNT = isLds; 503 let has_vdata = !not(!or(isLds, isLdsOpc)); 504 let mayLoad = 1; 505 let mayStore = isLds; 506 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 507 let tfe = isTFE; 508 let lds = isLds; 509 let elements = getMUBUFElements<vdata_vt>.ret; 510 let VALU = isLds; 511} 512 513class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : GCNPat < 514 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 515 (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset)) 516>; 517 518class MUBUF_Addr64_Load_Pat <Instruction inst, 519 ValueType load_vt = i32, 520 SDPatternOperator ld = null_frag> : GCNPat < 521 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 522 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)) 523>; 524 525multiclass MUBUF_Pseudo_Load_Pats_Common<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 526 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 527 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 528} 529 530multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag>{ 531 let SubtargetPredicate = HasUnrestrictedSOffset in { 532 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst, load_vt, ld>; 533 } 534 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst # "_VBUFFER", load_vt, ld>; 535} 536 537multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt, 538 bit TiedDest, bit isLds, bit isTFE, bit hasGFX12Enc> { 539 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 540 541 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>, 542 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 543 544 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>, 545 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 546 547 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 548 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 549 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 550 551 let DisableWQM = 1 in { 552 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 553 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 554 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 555 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 556 } 557} 558 559multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32, 560 bit TiedDest = 0, bit isLds = 0> { 561 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>; 562 defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>; 563 564 if !not(isLds) then { 565 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>; 566 defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>; 567 } 568} 569 570multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 571 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 572 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 573} 574 575multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 576 ValueType load_vt = i32, 577 bit TiedDest = 0, 578 bit isLds = 0, 579 bit isLdsOpc = 1> { 580 581 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 582 583 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 584 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 585 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 586 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 587 588 def _VBUFFER_OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 589 def _VBUFFER_OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 590 def _VBUFFER_IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 591 def _VBUFFER_BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 592} 593 594class MUBUF_Store_Pseudo <string opName, 595 int addrKind, 596 ValueType store_vt, 597 bit isTFE = 0, 598 bit hasGFX12Enc = 0, 599 list<dag> pattern=[], 600 // Workaround bug bz30254 601 int addrKindCopy = addrKind> 602 : MUBUF_Pseudo<opName, 603 (outs), 604 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret.RegClass], isTFE, hasGFX12Enc>.ret, 605 getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret, 606 pattern>, 607 MUBUF_SetupAddr<addrKindCopy> { 608 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") # 609 getAddrName<addrKindCopy>.ret; 610 let mayLoad = 0; 611 let mayStore = 1; 612 let elements = getMUBUFElements<store_vt>.ret; 613 let tfe = isTFE; 614} 615 616multiclass MUBUF_Pseudo_Store_Pats_Common<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 617 618 def : GCNPat < 619 (st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 620 (!cast<MUBUF_Pseudo>(BaseInst # _OFFSET) store_vt:$vdata, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 621 622 def : GCNPat < 623 (st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 624 (!cast<MUBUF_Pseudo>(BaseInst # _ADDR64) store_vt:$vdata, i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 625} 626 627multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 628 let SubtargetPredicate = HasUnrestrictedSOffset in { 629 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst, store_vt, st>; 630 } 631 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst # "_VBUFFER", store_vt, st>; 632} 633 634multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt, 635 bit isTFE, bit hasGFX12Enc> { 636 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 637 638 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasGFX12Enc>, 639 MUBUFAddr64Table<0, NAME>; 640 641 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasGFX12Enc>, 642 MUBUFAddr64Table<1, NAME>; 643 644 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasGFX12Enc>; 645 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasGFX12Enc>; 646 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasGFX12Enc>; 647 648 let DisableWQM = 1 in { 649 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasGFX12Enc>; 650 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasGFX12Enc>; 651 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasGFX12Enc>; 652 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasGFX12Enc>; 653 } 654} 655 656multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> { 657 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>; 658 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>; 659 660 defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>; 661 defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>; 662} 663 664class MUBUF_Pseudo_Store_Lds<string opName> 665 : MUBUF_Pseudo<opName, 666 (outs), 667 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, i1imm:$swz), 668 " $srsrc, $soffset$offset lds$cpol"> { 669 let LGKM_CNT = 1; 670 let mayLoad = 1; 671 let mayStore = 1; 672 673 let has_vdata = 0; 674 let has_vaddr = 0; 675 let lds = 1; 676 let VALU = 1; 677 678 let Uses = [EXEC, M0]; 679 let AsmMatchConverter = "cvtMubuf"; 680} 681 682class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, bit hasGFX12Enc, 683 list<RegisterClass> vaddrList=[]> { 684 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 685 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 686 687 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); 688 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); 689 dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 690 dag MainInputs = !con((ins SReg_128:$srsrc), SOffset, (ins offset:$offset)); 691 dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol), 692 (ins CPol_NonGLC_WithDefault:$cpol)); 693 694 dag ret = !con(Data, MainInputs, CPol); 695} 696 697class getMUBUFAtomicIns<int addrKind, 698 RegisterClass vdataClass, 699 bit vdata_in, 700 bit hasGFX12Enc, 701 // Workaround bug bz30254 702 RegisterClass vdataClassCopy=vdataClass> { 703 dag ret = 704 !if(!eq(addrKind, BUFAddrKind.Offset), 705 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc>.ret, 706 !if(!eq(addrKind, BUFAddrKind.OffEn), 707 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VGPR_32]>.ret, 708 !if(!eq(addrKind, BUFAddrKind.IdxEn), 709 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VGPR_32]>.ret, 710 !if(!eq(addrKind, BUFAddrKind.BothEn), 711 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VReg_64]>.ret, 712 !if(!eq(addrKind, BUFAddrKind.Addr64), 713 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VReg_64]>.ret, 714 (ins)))))); 715} 716 717class MUBUF_Atomic_Pseudo<string opName, 718 int addrKind, 719 dag outs, 720 dag ins, 721 string asmOps, 722 list<dag> pattern=[], 723 // Workaround bug bz30254 724 int addrKindCopy = addrKind> 725 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 726 MUBUF_SetupAddr<addrKindCopy> { 727 let mayStore = 1; 728 let mayLoad = 1; 729 let hasPostISelHook = 1; 730 let hasSideEffects = 1; 731 let DisableWQM = 1; 732 let has_glc = 0; 733 let has_dlc = 0; 734 let has_sccb = 1; 735 let AsmMatchConverter = "cvtMubufAtomic"; 736} 737 738class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 739 RegisterClass vdataClass, 740 bit hasGFX12Enc = 0, 741 list<dag> pattern=[], 742 // Workaround bug bz30254 743 int addrKindCopy = addrKind, 744 RegisterClass vdataClassCopy = vdataClass> 745 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 746 (outs), 747 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0, hasGFX12Enc>.ret, 748 getMUBUFAsmOps<addrKindCopy>.ret, 749 pattern>, 750 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 751 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 752 let glc_value = 0; 753 let dlc_value = 0; 754 let sccb_value = 0; 755 let IsAtomicNoRet = 1; 756} 757 758class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 759 RegisterClass vdataClass, 760 bit hasGFX12Enc = 0, 761 list<dag> pattern=[], 762 // Workaround bug bz30254 763 int addrKindCopy = addrKind, 764 RegisterClass vdataClassCopy = vdataClass, 765 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 766 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 767 (outs vdata_op:$vdata), 768 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1, hasGFX12Enc>.ret, 769 getMUBUFAsmOps<addrKindCopy>.ret, 770 pattern>, 771 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 772 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 773 let glc_value = 1; 774 let dlc_value = 0; 775 let sccb_value = 0; 776 let IsAtomicRet = 1; 777 let Constraints = "$vdata = $vdata_in"; 778 let DisableEncoding = "$vdata_in"; 779} 780 781multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 782 RegisterClass vdataClass, 783 ValueType vdataType> { 784 let FPAtomic = vdataType.isFP in { 785 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>, 786 MUBUFAddr64Table <0, NAME>; 787 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>, 788 MUBUFAddr64Table <1, NAME>; 789 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 790 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 791 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 792 793 def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>, 794 MUBUFAddr64Table <0, NAME # "_VBUFFER">; 795 def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>, 796 MUBUFAddr64Table <1, NAME # "_VBUFFER">; 797 def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 798 def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 799 def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 800 } 801} 802 803multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 804 RegisterClass vdataClass, 805 ValueType vdataType, 806 SDPatternOperator atomic> { 807 let FPAtomic = vdataType.isFP in { 808 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0, 809 [(set vdataType:$vdata, 810 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 811 vdataType:$vdata_in))]>, 812 MUBUFAddr64Table <0, NAME # "_RTN">; 813 814 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0, 815 [(set vdataType:$vdata, 816 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 817 vdataType:$vdata_in))]>, 818 MUBUFAddr64Table <1, NAME # "_RTN">; 819 820 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 821 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 822 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 823 824 def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1, 825 [(set vdataType:$vdata, 826 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 827 vdataType:$vdata_in))]>, 828 MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">; 829 830 def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1, 831 [(set vdataType:$vdata, 832 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 833 vdataType:$vdata_in))]>, 834 MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">; 835 836 def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 837 def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 838 def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 839 } 840} 841 842multiclass MUBUF_Pseudo_Atomics <string opName, 843 RegisterClass vdataClass, 844 ValueType vdataType, 845 SDPatternOperator atomic = null_frag> : 846 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 847 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 848 849 850//===----------------------------------------------------------------------===// 851// MUBUF Instructions 852//===----------------------------------------------------------------------===// 853 854defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 855 "buffer_load_format_x", f32 856>; 857defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 858 "buffer_load_format_xy", v2f32 859>; 860defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 861 "buffer_load_format_xyz", v3f32 862>; 863defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 864 "buffer_load_format_xyzw", v4f32 865>; 866defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 867 "buffer_store_format_x", f32 868>; 869defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 870 "buffer_store_format_xy", v2f32 871>; 872defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 873 "buffer_store_format_xyz", v3f32 874>; 875defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 876 "buffer_store_format_xyzw", v4f32 877>; 878 879let OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1 in { 880let TiedSourceNotRead = 1 in { 881 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 882 "buffer_load_format_d16_x", i32 883 >; 884 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 885 "buffer_load_format_d16_xy", v2i32 886 >; 887 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 888 "buffer_load_format_d16_xyz", v3i32 889 >; 890 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 891 "buffer_load_format_d16_xyzw", v4i32 892 >; 893} 894 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 895 "buffer_store_format_d16_x", i32 896 >; 897 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 898 "buffer_store_format_d16_xy", v2i32 899 >; 900 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 901 "buffer_store_format_d16_xyz", v3i32 902 >; 903 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 904 "buffer_store_format_d16_xyzw", v4i32 905 >; 906} // End OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1. 907 908let OtherPredicates = [HasPackedD16VMem], D16Buf = 1 in { 909let TiedSourceNotRead = 1 in { 910 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 911 "buffer_load_format_d16_x", f16 912 >; 913 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 914 "buffer_load_format_d16_xy", v2f16 915 >; 916 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 917 "buffer_load_format_d16_xyz", v3f16 918 >; 919 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 920 "buffer_load_format_d16_xyzw", v4f16 921 >; 922} 923 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 924 "buffer_store_format_d16_x", f16 925 >; 926 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 927 "buffer_store_format_d16_xy", v2f16 928 >; 929 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 930 "buffer_store_format_d16_xyz", v3f16 931 >; 932 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 933 "buffer_store_format_d16_xyzw", v4f16 934 >; 935} // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1. 936 937defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 938 "buffer_load_ubyte", i32 939>; 940defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 941 "buffer_load_sbyte", i32 942>; 943defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 944 "buffer_load_ushort", i32 945>; 946defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 947 "buffer_load_sshort", i32 948>; 949defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 950 "buffer_load_dword", i32 951>; 952defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 953 "buffer_load_dwordx2", v2i32 954>; 955defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 956 "buffer_load_dwordx3", v3i32 957>; 958defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 959 "buffer_load_dwordx4", v4i32 960>; 961 962defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 963 "buffer_load_lds_b32", i32 964>; 965defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 966 "buffer_load_lds_format_x", f32 967>; 968defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 969 "buffer_load_lds_i8", i32 970>; 971defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 972 "buffer_load_lds_i16", i32 973>; 974defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 975 "buffer_load_lds_u8", i32 976>; 977defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 978 "buffer_load_lds_u16", i32 979>; 980 981defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 982defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 983defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 984defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 985defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 986defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 987defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 988defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 989defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 990defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 991 992foreach vt = Reg32Types.types in { 993defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>; 994} 995 996foreach vt = VReg_64.RegTypes in { 997defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>; 998} 999 1000foreach vt = VReg_96.RegTypes in { 1001defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", vt, load_global>; 1002} 1003 1004foreach vt = VReg_128.RegTypes in { 1005defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", vt, load_global>; 1006} 1007 1008defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 1009 "buffer_store_byte", i32 1010>; 1011defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 1012 "buffer_store_short", i32 1013>; 1014defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 1015 "buffer_store_dword", i32 1016>; 1017defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 1018 "buffer_store_dwordx2", v2i32 1019>; 1020defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 1021 "buffer_store_dwordx3", v3i32 1022>; 1023defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 1024 "buffer_store_dwordx4", v4i32 1025>; 1026 1027defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_BYTE", i32, truncstorei8_global>; 1028defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>; 1029 1030foreach vt = Reg32Types.types in { 1031defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>; 1032} 1033 1034foreach vt = VReg_64.RegTypes in { 1035defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>; 1036} 1037 1038foreach vt = VReg_96.RegTypes in { 1039defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX3", vt, store_global>; 1040} 1041 1042foreach vt = VReg_128.RegTypes in { 1043defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX4", vt, store_global>; 1044} 1045 1046defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 1047 "buffer_atomic_swap", VGPR_32, i32 1048>; 1049defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 1050 "buffer_atomic_cmpswap", VReg_64, v2i32 1051>; 1052defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 1053 "buffer_atomic_add", VGPR_32, i32 1054>; 1055defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 1056 "buffer_atomic_sub", VGPR_32, i32 1057>; 1058defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 1059 "buffer_atomic_smin", VGPR_32, i32 1060>; 1061defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 1062 "buffer_atomic_umin", VGPR_32, i32 1063>; 1064defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 1065 "buffer_atomic_smax", VGPR_32, i32 1066>; 1067defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 1068 "buffer_atomic_umax", VGPR_32, i32 1069>; 1070defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_and", VGPR_32, i32 1072>; 1073defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 1074 "buffer_atomic_or", VGPR_32, i32 1075>; 1076defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1077 "buffer_atomic_xor", VGPR_32, i32 1078>; 1079defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1080 "buffer_atomic_inc", VGPR_32, i32 1081>; 1082defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1083 "buffer_atomic_dec", VGPR_32, i32 1084>; 1085defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1086 "buffer_atomic_swap_x2", VReg_64, i64 1087>; 1088defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1089 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1090>; 1091defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1092 "buffer_atomic_add_x2", VReg_64, i64 1093>; 1094defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1095 "buffer_atomic_sub_x2", VReg_64, i64 1096>; 1097defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1098 "buffer_atomic_smin_x2", VReg_64, i64 1099>; 1100defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1101 "buffer_atomic_umin_x2", VReg_64, i64 1102>; 1103defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1104 "buffer_atomic_smax_x2", VReg_64, i64 1105>; 1106defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1107 "buffer_atomic_umax_x2", VReg_64, i64 1108>; 1109defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1110 "buffer_atomic_and_x2", VReg_64, i64 1111>; 1112defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1113 "buffer_atomic_or_x2", VReg_64, i64 1114>; 1115defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1116 "buffer_atomic_xor_x2", VReg_64, i64 1117>; 1118defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1119 "buffer_atomic_inc_x2", VReg_64, i64 1120>; 1121defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1122 "buffer_atomic_dec_x2", VReg_64, i64 1123>; 1124 1125let OtherPredicates = [HasGFX10_BEncoding] in { 1126 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics < 1127 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1128 >; 1129} 1130 1131let SubtargetPredicate = isGFX8GFX9 in { 1132def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1133} 1134 1135let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1136/* 1137defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1138defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1139*/ 1140 1141def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1142 int_amdgcn_buffer_wbinvl1_sc>; 1143} 1144 1145let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1146 1147defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1148 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1149>; 1150defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1151 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1152>; 1153defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1154 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1155>; 1156 1157} 1158 1159let SubtargetPredicate = isGFX6GFX7GFX10 in { 1160 1161defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1162 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1163>; 1164defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1165 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1166>; 1167defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1168 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1169>; 1170 1171} 1172 1173let SubtargetPredicate = HasD16LoadStore in { 1174let TiedSourceNotRead = 1 in { 1175 1176defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1177 "buffer_load_ubyte_d16", i32, 1 1178>; 1179 1180defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1181 "buffer_load_ubyte_d16_hi", i32, 1 1182>; 1183 1184defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1185 "buffer_load_sbyte_d16", i32, 1 1186>; 1187 1188defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1189 "buffer_load_sbyte_d16_hi", i32, 1 1190>; 1191 1192defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1193 "buffer_load_short_d16", i32, 1 1194>; 1195 1196defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1197 "buffer_load_short_d16_hi", i32, 1 1198>; 1199 1200defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1201 "buffer_load_format_d16_hi_x", i32 1202>; 1203} // End TiedSourceNotRead 1204 1205defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1206 "buffer_store_byte_d16_hi", i32 1207>; 1208 1209defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1210 "buffer_store_short_d16_hi", i32 1211>; 1212 1213defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1214 "buffer_store_format_d16_hi_x", i32 1215>; 1216 1217} // End HasD16LoadStore 1218 1219let SubtargetPredicate = isNotGFX12Plus in 1220def BUFFER_WBINVL1 : MUBUF_Invalidate < 1221 "buffer_wbinvl1", int_amdgcn_buffer_wbinvl1 1222>; 1223 1224let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1225defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1226 "buffer_atomic_add_f32", VGPR_32, f32 1227>; 1228 1229let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in 1230defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1231 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1232>; 1233 1234let OtherPredicates = [HasAtomicFaddRtnInsts] in 1235defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1236 "buffer_atomic_add_f32", VGPR_32, f32, null_frag 1237>; 1238 1239let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in 1240defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1241 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag 1242>; 1243 1244let SubtargetPredicate = isGFX12Plus in { 1245defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Pseudo_Atomics < 1246 "buffer_atomic_cond_sub_u32", VGPR_32, i32 1247>; 1248 1249let FPAtomic = 1 in 1250defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Pseudo_Atomics < 1251 "buffer_atomic_pk_add_bf16", VGPR_32, v2bf16 1252>; 1253} 1254 1255//===----------------------------------------------------------------------===// 1256// MTBUF Instructions 1257//===----------------------------------------------------------------------===// 1258 1259defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1260defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1261defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1262defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1263defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1264defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1265defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1266defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1267 1268let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1269let TiedSourceNotRead = 1 in { 1270 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1271 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1272 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1273 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1274} 1275 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1276 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1277 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1278 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1279} // End HasUnpackedD16VMem. 1280 1281let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1282let TiedSourceNotRead = 1 in { 1283 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1284 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1285 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1286 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1287} 1288 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1289 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1290 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1291 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1292} // End HasPackedD16VMem. 1293 1294let SubtargetPredicate = isGFX7Plus in { 1295 1296//===----------------------------------------------------------------------===// 1297// Instruction definitions for CI and newer. 1298//===----------------------------------------------------------------------===// 1299 1300def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1301 int_amdgcn_buffer_wbinvl1_vol>; 1302 1303} // End let SubtargetPredicate = isGFX7Plus 1304 1305let SubtargetPredicate = isGFX90APlus in { 1306 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1307 let has_glc = 1; 1308 let has_sccb = 1; 1309 let InOperandList = (ins CPol_0:$cpol); 1310 let AsmOperands = "$cpol"; 1311 } 1312 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1313 let SubtargetPredicate = isGFX90AOnly; 1314 } 1315 1316 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1317 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1318 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1319} // End SubtargetPredicate = isGFX90APlus 1320 1321def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1322 let SubtargetPredicate = isGFX940Plus; 1323 let has_glc = 1; 1324 let has_sccb = 1; 1325 let InOperandList = (ins CPol_0:$cpol); 1326 let AsmOperands = "$cpol"; 1327} 1328 1329def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1330def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1331 1332//===----------------------------------------------------------------------===// 1333// MUBUF Patterns 1334//===----------------------------------------------------------------------===// 1335 1336//===----------------------------------------------------------------------===// 1337// buffer_load/store_format patterns 1338//===----------------------------------------------------------------------===// 1339 1340multiclass MUBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1341 string opcode, ValueType memoryVt = vt> { 1342 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1343 1344 def : GCNPat< 1345 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1346 timm:$auxiliary, 0)), 1347 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1348 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1349 >; 1350 1351 def : GCNPat< 1352 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1353 timm:$auxiliary, 0)), 1354 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1355 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1356 >; 1357 1358 def : GCNPat< 1359 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1360 timm:$auxiliary, timm)), 1361 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1362 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1363 >; 1364 1365 def : GCNPat< 1366 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1367 timm:$auxiliary, timm)), 1368 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1369 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1370 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1371 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1372 >; 1373} 1374 1375multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1376 string opcode, ValueType memoryVt = vt>{ 1377 let SubtargetPredicate = HasUnrestrictedSOffset in { 1378 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1379 } 1380 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1381} 1382 1383defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1384defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1385defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1386defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1387defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1388defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1389defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1390defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1391 1392defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">; 1393defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">; 1394defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">; 1395defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">; 1396 1397let OtherPredicates = [HasUnpackedD16VMem] in { 1398 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1399 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1400 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1401 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1402 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1403 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1404} // End HasUnpackedD16VMem. 1405 1406let OtherPredicates = [HasPackedD16VMem] in { 1407 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1408 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1409 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1410 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1411 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1412 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1413 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1414 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1415 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1416} // End HasPackedD16VMem. 1417 1418defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1419defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1420defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1421defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1422defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1423defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1424defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1425defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1426defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1427defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1428defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1429defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1430defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1431defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1432defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1433defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1434 1435multiclass MUBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1436 string opcode, ValueType memoryVt = vt> { 1437 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1438 1439 def : GCNPat< 1440 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1441 timm:$auxiliary, 0), 1442 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1443 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1444 >; 1445 1446 def : GCNPat< 1447 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1448 timm:$auxiliary, 0), 1449 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1450 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1451 >; 1452 1453 def : GCNPat< 1454 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1455 timm:$auxiliary, timm), 1456 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1457 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1458 >; 1459 1460 def : GCNPat< 1461 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1462 timm:$auxiliary, timm), 1463 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1464 getVregSrcForVT<vt>.ret:$vdata, 1465 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1466 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary), 1467 (extract_swz $auxiliary)) 1468 >; 1469} 1470 1471multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1472 string opcode, ValueType memoryVt = vt> { 1473 let SubtargetPredicate = HasUnrestrictedSOffset in { 1474 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1475 } 1476 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1477} 1478 1479defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1480defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1481defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1482defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1483defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1484defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1485defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1486defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1487 1488let OtherPredicates = [HasUnpackedD16VMem] in { 1489 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1490 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1491 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1492 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1493 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1494 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1495} // End HasUnpackedD16VMem. 1496 1497let OtherPredicates = [HasPackedD16VMem] in { 1498 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1499 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1500 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1501 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1502 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1503 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1504 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1505 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1506 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1507} // End HasPackedD16VMem. 1508 1509defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1510defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1511defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1512defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1513defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1514defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1515defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1516defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1517defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1518defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1519defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1520defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1521defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1522defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1523 1524//===----------------------------------------------------------------------===// 1525// buffer_atomic patterns 1526//===----------------------------------------------------------------------===// 1527 1528multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1529 foreach RtnMode = ["ret", "noret"] in { 1530 1531 defvar Op = !cast<SDPatternOperator>(OpPrefix 1532 # !if(!eq(RtnMode, "ret"), "", "_noret") 1533 # !if(isIntr, "", "_" # vt.Size)); 1534 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1535 1536 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1537 def : GCNPat< 1538 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)), 1539 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1540 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1541 >; 1542 1543 def : GCNPat< 1544 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1545 vt:$vdata_in)), 1546 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1547 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1548 >; 1549 } // end let AddedComplexity 1550 1551 } // end foreach RtnMode 1552} 1553 1554multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1555 let SubtargetPredicate = HasUnrestrictedSOffset in { 1556 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst, isIntr>; 1557 } 1558 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", isIntr>; 1559} 1560 1561multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1562 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1563} 1564 1565multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1566 foreach RtnMode = ["ret", "noret"] in { 1567 1568 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1569 # !if(!eq(RtnMode, "ret"), "", "_noret") 1570 # "_" # vt.Size); 1571 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1572 defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass; 1573 1574 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1575 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1576 data_vt_RC:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1577 offset:$offset); 1578 def : GCNPat< 1579 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)), 1580 !if(!eq(RtnMode, "ret"), 1581 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, data_vt_RC)), 1582 !if(!eq(vt, i32), sub0, sub0_sub1)), 1583 OffsetResDag) 1584 >; 1585 1586 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1587 data_vt_RC:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1588 SCSrc_b32:$soffset, offset:$offset); 1589 def : GCNPat< 1590 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1591 data_vt:$vdata_in)), 1592 !if(!eq(RtnMode, "ret"), 1593 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, data_vt_RC)), 1594 !if(!eq(vt, i32), sub0, sub0_sub1)), 1595 Addr64ResDag) 1596 >; 1597 } // end let AddedComplexity 1598 1599 } // end foreach RtnMode 1600} 1601 1602multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1603 let SubtargetPredicate = HasUnrestrictedSOffset in { 1604 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1605 } 1606 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1607} 1608 1609 1610foreach Ty = [i32, i64] in { 1611 1612defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1613 1614defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1615defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1616defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1617defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1618defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1619defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1620defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1621defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1622defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1623defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1624defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1625defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1626 1627} // end foreach Ty 1628 1629defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1630defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1631 1632multiclass SIBufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, 1633 list<string> RtnModes = ["ret", "noret"]> { 1634 foreach RtnMode = RtnModes in { 1635 1636 defvar Op = !cast<SDPatternOperator>(OpPrefix 1637 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1638 1639 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1640 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1641 (extract_cpol_set_glc $auxiliary), (extract_cpol $auxiliary)); 1642 1643 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1644 def : GCNPat< 1645 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1646 timm:$offset, timm:$auxiliary, 0)), 1647 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1648 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1649 timm:$offset, CachePolicy) 1650 >; 1651 1652 def : GCNPat< 1653 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), 1654 timm:$offset, timm:$auxiliary, timm)), 1655 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1656 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1657 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1658 >; 1659 1660 def : GCNPat< 1661 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1662 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, 0)), 1663 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1664 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1665 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1666 >; 1667 1668 def : GCNPat< 1669 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1670 (BUFSOffset i32:$soffset), timm:$offset, timm:$auxiliary, timm)), 1671 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1672 getVregSrcForVT<vt>.ret:$vdata_in, 1673 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1674 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1675 >; 1676 } // end let AddedComplexity 1677 1678 } // end foreach RtnMode 1679} 1680 1681multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1682 list<string> RtnModes = ["ret", "noret"]> { 1683 let SubtargetPredicate = HasUnrestrictedSOffset in { 1684 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst, RtnModes>; 1685 } 1686 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", RtnModes>; 1687} 1688 1689defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1690defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1691defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1692defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1693defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1694defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1695defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1696defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1697defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1698defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1699defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1700defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1701defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1702defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1703defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1704defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1705defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1706defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1707defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1708defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1709defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1710defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1711defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1712defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1713defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1714defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1715 1716let OtherPredicates = [HasAtomicCSubNoRtnInsts] in 1717defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>; 1718 1719let SubtargetPredicate = isGFX12Plus in { 1720 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd_bf16", v2bf16, "BUFFER_ATOMIC_PK_ADD_BF16_VBUFFER">; 1721 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["ret"]>; 1722 1723 let OtherPredicates = [HasAtomicCSubNoRtnInsts] in 1724 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_cond_sub_u32", i32, "BUFFER_ATOMIC_COND_SUB_U32_VBUFFER", ["noret"]>; 1725} 1726 1727let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1728 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1729 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1730} 1731let SubtargetPredicate = isGFX6GFX7GFX10 in { 1732 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1733 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1734} 1735 1736class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1737 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1738 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> { 1739 let HasNoUse = true; 1740} 1741 1742multiclass BufferAtomicPatterns_NO_RTN_Common<SDPatternOperator name, ValueType vt, 1743 string opcode> { 1744 def : GCNPat< 1745 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1746 0, (BUFSOffset i32:$soffset), timm:$offset, 1747 timm:$auxiliary, 0), 1748 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1749 timm:$offset, (extract_cpol $auxiliary)) 1750 >; 1751 1752 def : GCNPat< 1753 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1754 0, (BUFSOffset i32:$soffset), timm:$offset, 1755 timm:$auxiliary, timm), 1756 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1757 timm:$offset, (extract_cpol $auxiliary)) 1758 >; 1759 1760 def : GCNPat< 1761 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1762 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1763 timm:$auxiliary, 0), 1764 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1765 timm:$offset, (extract_cpol $auxiliary)) 1766 >; 1767 1768 def : GCNPat< 1769 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1770 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1771 timm:$auxiliary, timm), 1772 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1773 getVregSrcForVT<vt>.ret:$vdata_in, 1774 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1775 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary)) 1776 >; 1777} 1778 1779multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1780 string opcode> { 1781 let SubtargetPredicate = HasUnrestrictedSOffset in { 1782 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode>; 1783 } 1784 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode # "_VBUFFER">; 1785} 1786 1787let OtherPredicates = [HasAtomicFaddNoRtnInsts] in 1788 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>; 1789 1790let OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] in { 1791 let SubtargetPredicate = isGFX9Only in 1792 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>; 1793 1794 let SubtargetPredicate = isGFX12Plus in 1795 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16_VBUFFER", ["noret"]>; 1796} // End OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] 1797 1798let OtherPredicates = [HasAtomicFaddRtnInsts] in 1799 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>; 1800 1801let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in { 1802 let SubtargetPredicate = isGFX9Only in 1803 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>; 1804 1805 let SubtargetPredicate = isGFX12Plus in 1806 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16_VBUFFER", ["ret"]>; 1807} // End OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] 1808 1809let OtherPredicates = [isGFX90APlus] in { 1810 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1811 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1812 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1813} // End SubtargetPredicate = isGFX90APlus 1814 1815multiclass SIBufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1816 foreach RtnMode = ["ret", "noret"] in { 1817 defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1818 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1819 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1820 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1821 (extract_cpol_set_glc $auxiliary), 1822 (extract_cpol $auxiliary)); 1823 defvar SrcRC = getVregSrcForVT<vt>.ret; 1824 defvar DataRC = getVregSrcForVT<data_vt>.ret.RegClass; 1825 defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1); 1826 defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3); 1827 1828 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1829 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1830 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1831 def : GCNPat< 1832 (vt (Op 1833 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1834 timm:$offset, timm:$auxiliary, 0)), 1835 !if(!eq(RtnMode, "ret"), 1836 (EXTRACT_SUBREG OffsetResDag, SubLo), 1837 OffsetResDag) 1838 >; 1839 1840 defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1841 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1842 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1843 CachePolicy); 1844 def : GCNPat< 1845 (vt (Op 1846 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1847 0, (BUFSOffset i32:$soffset), timm:$offset, 1848 timm:$auxiliary, timm)), 1849 !if(!eq(RtnMode, "ret"), 1850 (EXTRACT_SUBREG IdxenResDag, SubLo), 1851 IdxenResDag) 1852 >; 1853 1854 defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1855 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1856 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1857 CachePolicy); 1858 def : GCNPat< 1859 (vt (Op 1860 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 1861 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1862 timm:$auxiliary, 0)), 1863 !if(!eq(RtnMode, "ret"), 1864 (EXTRACT_SUBREG OffenResDag, SubLo), 1865 OffenResDag) 1866 >; 1867 1868 defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1869 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1870 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1871 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1872 def : GCNPat< 1873 (vt (Op 1874 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1875 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1876 timm:$auxiliary, timm)), 1877 !if(!eq(RtnMode, "ret"), 1878 (EXTRACT_SUBREG BothenResDag, SubLo), 1879 BothenResDag) 1880 >; 1881 } // end foreach RtnMode 1882} 1883 1884multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1885 let SubtargetPredicate = HasUnrestrictedSOffset in { 1886 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1887 } 1888 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1889} 1890 1891defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1892defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1893 1894class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1895 PatFrag constant_ld> : GCNPat < 1896 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1897 i32:$offset))), 1898 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1899 >; 1900 1901multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1902 ValueType vt, PatFrag atomic_ld> { 1903 def : GCNPat < 1904 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 1905 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1906 >; 1907 1908 def : GCNPat < 1909 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), 1910 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1911 >; 1912} 1913 1914let SubtargetPredicate = isGFX6GFX7 in { 1915def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1916def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1917def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1918def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1919def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1920def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1921 1922defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1923defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1924} // End SubtargetPredicate = isGFX6GFX7 1925 1926multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt, 1927 PatFrag ld> { 1928 def : GCNPat < 1929 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1930 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $srsrc, $soffset, $offset) 1931 >; 1932} 1933 1934multiclass MUBUFLoad_PatternOffset <string Instr, ValueType vt, 1935 PatFrag ld> { 1936 let SubtargetPredicate = HasUnrestrictedSOffset in { 1937 defm : MUBUFLoad_PatternOffset_Common<Instr, vt, ld>; 1938 } 1939 defm : MUBUFLoad_PatternOffset_Common<Instr # "_VBUFFER", vt, ld>; 1940} 1941 1942let OtherPredicates = [Has16BitInsts] in { 1943 1944defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_constant>; 1945defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_constant>; 1946defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_constant>; 1947defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_global>; 1948defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_global>; 1949defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_global>; 1950 1951defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_USHORT", i16, load_global>; 1952 1953} // End OtherPredicates = [Has16BitInsts] 1954 1955multiclass MUBUFScratchLoadPat_Common <string Instr, 1956 ValueType vt, PatFrag ld> { 1957 def : GCNPat < 1958 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1959 i32:$soffset, i32:$offset))), 1960 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, 0, 0) 1961 >; 1962 1963 def : GCNPat < 1964 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1965 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, 0, 0) 1966 >; 1967} 1968 1969multiclass MUBUFScratchLoadPat <string Instr, 1970 ValueType vt, PatFrag ld> { 1971 let SubtargetPredicate = HasUnrestrictedSOffset in { 1972 defm : MUBUFScratchLoadPat_Common<Instr, vt, ld>; 1973 } 1974 defm : MUBUFScratchLoadPat_Common<Instr # "_VBUFFER", vt, ld>; 1975} 1976 1977// XXX - Is it possible to have a complex pattern in a PatFrag? 1978multiclass MUBUFScratchLoadPat_D16_Common <string Instr, 1979 ValueType vt, PatFrag ld_frag> { 1980 def : GCNPat < 1981 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in), 1982 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, $in) 1983 >; 1984 1985 def : GCNPat < 1986 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in), 1987 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, $in) 1988 >; 1989} 1990 1991multiclass MUBUFScratchLoadPat_D16 <string Instr, 1992 ValueType vt, PatFrag ld_frag> { 1993 let SubtargetPredicate = HasUnrestrictedSOffset in { 1994 defm : MUBUFScratchLoadPat_D16_Common<Instr, vt, ld_frag>; 1995 } 1996 defm : MUBUFScratchLoadPat_D16_Common<Instr # "_VBUFFER", vt, ld_frag>; 1997} 1998 1999let OtherPredicates = [DisableFlatScratch] in { 2000defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i32, sextloadi8_private>; 2001defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, extloadi8_private>; 2002defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, zextloadi8_private>; 2003defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i16, sextloadi8_private>; 2004defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, extloadi8_private>; 2005defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, zextloadi8_private>; 2006defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SSHORT", i32, sextloadi16_private>; 2007defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, extloadi16_private>; 2008defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, zextloadi16_private>; 2009defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i16, load_private>; 2010 2011foreach vt = Reg32Types.types in { 2012defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORD", vt, load_private>; 2013} 2014defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX2", v2i32, load_private>; 2015defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX3", v3i32, load_private>; 2016defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX4", v4i32, load_private>; 2017 2018let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 2019defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2i16, load_d16_hi_private>; 2020defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2i16, az_extloadi8_d16_hi_private>; 2021defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2i16, sextloadi8_d16_hi_private>; 2022defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2f16, load_d16_hi_private>; 2023defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2f16, az_extloadi8_d16_hi_private>; 2024defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2f16, sextloadi8_d16_hi_private>; 2025 2026defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2i16, load_d16_lo_private>; 2027defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2i16, az_extloadi8_d16_lo_private>; 2028defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2i16, sextloadi8_d16_lo_private>; 2029defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2f16, load_d16_lo_private>; 2030defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2f16, az_extloadi8_d16_lo_private>; 2031defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2f16, sextloadi8_d16_lo_private>; 2032} 2033 2034} // End OtherPredicates = [DisableFlatScratch] 2035 2036multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 2037 ValueType vt, PatFrag atomic_st> { 2038 def : GCNPat < 2039 (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 2040 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 2041 >; 2042 2043 def : GCNPat < 2044 (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)), 2045 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 2046 >; 2047} 2048let SubtargetPredicate = isGFX6GFX7 in { 2049defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 2050defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 2051defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 2052defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 2053defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 2054defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 2055} // End Predicates = isGFX6GFX7 2056 2057 2058multiclass MUBUFStore_PatternOffset_Common <string Instr, ValueType vt, 2059 PatFrag st> { 2060 2061 def : GCNPat < 2062 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 2063 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $vdata, $srsrc, $soffset, $offset) 2064 >; 2065} 2066 2067multiclass MUBUFStore_PatternOffset <string Instr, ValueType vt, 2068 PatFrag st> { 2069 let SubtargetPredicate = HasUnrestrictedSOffset in { 2070 defm : MUBUFStore_PatternOffset_Common<Instr, vt, st>; 2071 } 2072 defm : MUBUFStore_PatternOffset_Common<Instr # "_VBUFFER", vt, st>; 2073} 2074 2075defm : MUBUFStore_PatternOffset <"BUFFER_STORE_BYTE", i16, truncstorei8_global>; 2076defm : MUBUFStore_PatternOffset <"BUFFER_STORE_SHORT", i16, store_global>; 2077 2078multiclass MUBUFScratchStorePat_Common <string Instr, 2079 ValueType vt, PatFrag st, 2080 RegisterClass rc = VGPR_32> { 2081 def : GCNPat < 2082 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 2083 i32:$soffset, i32:$offset)), 2084 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) 2085 >; 2086 2087 def : GCNPat < 2088 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 2089 i32:$offset)), 2090 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) rc:$value, $srsrc, $soffset, $offset, 0, 0) 2091 >; 2092} 2093 2094multiclass MUBUFScratchStorePat <string Instr, 2095 ValueType vt, PatFrag st, 2096 RegisterClass rc = VGPR_32> { 2097 let SubtargetPredicate = HasUnrestrictedSOffset in { 2098 defm : MUBUFScratchStorePat_Common<Instr, vt, st, rc>; 2099 } 2100 defm : MUBUFScratchStorePat_Common<Instr # "_VBUFFER", vt, st, rc>; 2101} 2102 2103let OtherPredicates = [DisableFlatScratch] in { 2104defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i32, truncstorei8_private>; 2105defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i32, truncstorei16_private>; 2106defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i16, truncstorei8_private>; 2107defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i16, store_private>; 2108 2109foreach vt = Reg32Types.types in { 2110defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORD", vt, store_private>; 2111} 2112 2113defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX2", v2i32, store_private, VReg_64>; 2114defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX3", v3i32, store_private, VReg_96>; 2115defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX4", v4i32, store_private, VReg_128>; 2116 2117 2118let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 2119 // Hiding the extract high pattern in the PatFrag seems to not 2120 // automatically increase the complexity. 2121let AddedComplexity = 1 in { 2122defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT_D16_HI", i32, store_hi16_private>; 2123defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE_D16_HI", i32, truncstorei8_hi16_private>; 2124} 2125} 2126} // End OtherPredicates = [DisableFlatScratch] 2127 2128//===----------------------------------------------------------------------===// 2129// MTBUF Patterns 2130//===----------------------------------------------------------------------===// 2131 2132//===----------------------------------------------------------------------===// 2133// tbuffer_load/store_format patterns 2134//===----------------------------------------------------------------------===// 2135 2136multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2137 string opcode, ValueType memoryVt = vt> { 2138 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 2139 2140 def : GCNPat< 2141 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2142 timm:$format, timm:$auxiliary, 0)), 2143 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2144 (as_i8timm $format), 2145 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2146 >; 2147 2148 def : GCNPat< 2149 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2150 timm:$format, timm:$auxiliary, timm)), 2151 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2152 (as_i8timm $format), 2153 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2154 >; 2155 2156 def : GCNPat< 2157 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2158 timm:$format, timm:$auxiliary, 0)), 2159 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2160 (as_i8timm $format), 2161 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2162 >; 2163 2164 def : GCNPat< 2165 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2166 timm:$format, timm:$auxiliary, timm)), 2167 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 2168 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2169 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2170 (as_i8timm $format), 2171 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2172 >; 2173} 2174 2175multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 2176 string opcode, ValueType memoryVt = vt> { 2177 let SubtargetPredicate = HasUnrestrictedSOffset in { 2178 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2179 } 2180 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2181} 2182 2183defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 2184defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 2185defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 2186defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 2187defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 2188defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 2189defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 2190defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 2191 2192let OtherPredicates = [HasUnpackedD16VMem] in { 2193 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2194 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2195 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 2196 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 2197 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 2198} // End HasUnpackedD16VMem. 2199 2200let OtherPredicates = [HasPackedD16VMem] in { 2201 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 2202 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 2203 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 2204 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 2205 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 2206} // End HasPackedD16VMem. 2207 2208multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2209 string opcode, ValueType memoryVt = vt> { 2210 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 2211 2212 def : GCNPat< 2213 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2214 timm:$format, timm:$auxiliary, 0), 2215 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 2216 timm:$offset, (as_i8timm $format), 2217 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2218 >; 2219 2220 def : GCNPat< 2221 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2222 timm:$format, timm:$auxiliary, timm), 2223 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 2224 timm:$offset, (as_i8timm $format), 2225 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2226 >; 2227 2228 def : GCNPat< 2229 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2230 timm:$format, timm:$auxiliary, 0), 2231 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2232 timm:$offset, (as_i8timm $format), 2233 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2234 >; 2235 2236 def : GCNPat< 2237 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), 2238 timm:$offset, timm:$format, timm:$auxiliary, timm), 2239 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2240 getVregSrcForVT<vt>.ret:$vdata, 2241 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2242 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), 2243 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2244 >; 2245} 2246 2247multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 2248 string opcode, ValueType memoryVt = vt> { 2249 let SubtargetPredicate = HasUnrestrictedSOffset in { 2250 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2251 } 2252 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2253} 2254 2255defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2256defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2257defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2258defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2259defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2260defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2261defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2262defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2263 2264let OtherPredicates = [HasUnpackedD16VMem] in { 2265 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2266 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2267 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2268 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2269 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2270} // End HasUnpackedD16VMem. 2271 2272let OtherPredicates = [HasPackedD16VMem] in { 2273 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2274 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2275 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2276 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2277 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2278} // End HasPackedD16VMem. 2279 2280//===----------------------------------------------------------------------===// 2281// Target-specific instruction encodings. 2282//===----------------------------------------------------------------------===// 2283 2284//===----------------------------------------------------------------------===// 2285// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2286//===----------------------------------------------------------------------===// 2287 2288class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2289 string real_name = ps.Mnemonic> : 2290 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2291 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2292 let Inst{31-26} = 0x38; 2293 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2294 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2295 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2296 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2297} 2298 2299class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2300 string real_name = ps.Mnemonic> : 2301 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2302 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2303 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2304 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2305 let Inst{25-18} = op; 2306 let Inst{53} = ps.tfe; 2307 let Inst{54} = ps.offen; 2308 let Inst{55} = ps.idxen; 2309 let SubtargetPredicate = isGFX11Only; 2310} 2311 2312class Base_MUBUF_Real_Atomic_gfx11<bits<8> op, MUBUF_Pseudo ps, 2313 string real_name> : 2314 MUBUF_Real_gfx11<op, ps, real_name> { 2315 let Inst{13} = cpol{CPolBit.DLC}; 2316} 2317 2318class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2319 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2320 let Inst{12} = ps.offen; 2321 let Inst{13} = ps.idxen; 2322 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2323 let Inst{16} = ps.lds; 2324 let Inst{24-18} = op; 2325 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2326 let Inst{55} = ps.tfe; 2327} 2328 2329class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2330 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2331 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2332 let Inst{25} = op{7}; 2333 let SubtargetPredicate = isGFX10Only; 2334} 2335 2336class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2337 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2338 let Inst{15} = ps.addr64; 2339 let SubtargetPredicate = isGFX6GFX7; 2340} 2341 2342//===----------------------------------------------------------------------===// 2343// Base ENC_VBUFFER for GFX12. 2344//===----------------------------------------------------------------------===// 2345 2346class VBUFFER_Real <BUF_Pseudo ps, string real_name = ps.Mnemonic> : 2347 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 { 2348 2349 let isPseudo = 0; 2350 let isCodeGenOnly = 0; 2351 2352 let VM_CNT = 1; 2353 let EXP_CNT = 1; 2354 2355 // copy relevant pseudo op flags 2356 let SubtargetPredicate = ps.SubtargetPredicate; 2357 let AsmMatchConverter = ps.AsmMatchConverter; 2358 let OtherPredicates = ps.OtherPredicates; 2359 let Constraints = ps.Constraints; 2360 let DisableEncoding = ps.DisableEncoding; 2361 let TSFlags = ps.TSFlags; 2362 let UseNamedOperandTable = ps.UseNamedOperandTable; 2363 let SchedRW = ps.SchedRW; 2364 let mayLoad = ps.mayLoad; 2365 let mayStore = ps.mayStore; 2366 let IsAtomicRet = ps.IsAtomicRet; 2367 let IsAtomicNoRet = ps.IsAtomicNoRet; 2368 let VALU = ps.VALU; 2369 let LGKM_CNT = ps.LGKM_CNT; 2370 2371 bits<24> offset; 2372 bits<8> vaddr; 2373 bits<10> vdata; 2374 2375 bits<7> srsrc; 2376 bits<7> soffset; 2377 bits<6> cpol; 2378 2379 let Inst{95-72} = !if(ps.has_offset, offset, ?); 2380 let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?); 2381 let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?); 2382 2383 let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?); 2384 let Inst{49-48} = 0b00; 2385 let Inst{6-0} = !if(ps.has_soffset, soffset, ?); 2386 let Inst{22} = ps.tfe; 2387 let Inst{62} = ps.offen; 2388 let Inst{63} = ps.idxen; 2389 2390 let Inst{54-53} = cpol{2-1}; // th{2-1} 2391 let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0} 2392 let Inst{51-50} = cpol{4-3}; // scope 2393 2394 let Inst{31-26} = 0b110001; 2395} 2396 2397class VBUFFER_MUBUF_Real_gfx12<bits<8> op, MUBUF_Pseudo ps, 2398 string real_name = ps.Mnemonic> : 2399 VBUFFER_Real<ps, real_name>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> { 2400 2401 let MUBUF = 1; 2402 2403 // Set the last bit of format to 1 to avoid round-trip issues, as some tools 2404 // print BUF_FMT_INVALID for format 0. 2405 let Inst{55} = 0b1; 2406 let Inst{21-14} = op; 2407 let SubtargetPredicate = isGFX12Only; 2408} 2409 2410class VBUFFER_MTBUF_Real_gfx12<bits<4> op, MTBUF_Pseudo ps, 2411 string real_name = ps.Mnemonic> : 2412 VBUFFER_Real<ps, real_name>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> { 2413 2414 let MTBUF = 1; 2415 2416 bits<7> format; 2417 2418 let Inst{17-14} = op; 2419 let Inst{21-18} = 0b1000; 2420 let Inst{61-55} = format; 2421} 2422 2423//===----------------------------------------------------------------------===// 2424// MUBUF - GFX11, GFX12. 2425//===----------------------------------------------------------------------===// 2426 2427// Shortcut to default Mnemonic from MUBUF_Pseudo. Hides the cast to the 2428// specific pseudo (bothen in this case) since any of them will work. 2429class get_MUBUF_ps<string name> { 2430 string Mnemonic = !cast<MUBUF_Pseudo>(name # "_BOTHEN").Mnemonic; 2431} 2432 2433// gfx11 instruction that accept both old and new assembler name. 2434class Mnem_gfx11_gfx12 <string mnemonic, string real_name> : 2435 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>; 2436 2437class Mnem_gfx11 <string mnemonic, string real_name> : 2438 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Only]>; 2439 2440class Mnem_gfx12 <string mnemonic, string real_name> : 2441 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX12Plus]>; 2442 2443class MUBUF_Real_gfx11_impl<bits<8> op, string ps_name, string real_name> : 2444 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2445 2446class VBUFFER_MUBUF_Real_gfx12_impl<bits<8> op, string ps_name, string real_name> : 2447 VBUFFER_MUBUF_Real_gfx12<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2448 2449multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<bits<8> op, string real_name> { 2450 let DecoderNamespace = "GFX11" in { 2451 def _BOTHEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_BOTHEN", real_name>; 2452 def _IDXEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_IDXEN", real_name>; 2453 def _OFFEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFEN", real_name>; 2454 def _OFFSET_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFSET", real_name>; 2455 } 2456} 2457 2458multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2<bits<8> op, string real_name> { 2459 let DecoderNamespace = "GFX12" in { 2460 def _BOTHEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN", real_name>; 2461 def _IDXEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_IDXEN", real_name>; 2462 def _OFFEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_OFFEN", real_name>; 2463 def _OFFSET_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_OFFSET", real_name>; 2464 } 2465} 2466 2467multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<bits<8> op, string real_name> : 2468 MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>, 2469 MUBUF_Real_AllAddr_gfx12_Renamed_Impl2<op, real_name>; 2470 2471multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name, 2472 bit hasTFE = 1> { 2473 defm NAME : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2474 if hasTFE then 2475 defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2476} 2477 2478multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<bits<8> op, string real_name, 2479 bit hasTFE = 1> { 2480 defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<op, real_name>; 2481 if hasTFE then 2482 defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<op, real_name>; 2483} 2484 2485// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions. 2486multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> : 2487 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>; 2488 2489multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op, bit hasTFE = 1> : 2490 MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>; 2491 2492multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<bits<8> op, string real_name> : 2493 MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, real_name> { 2494 def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2495} 2496 2497class MUBUF_Real_Atomic_gfx11_impl<bits<8> op, string ps_name, 2498 string real_name> : 2499 Base_MUBUF_Real_Atomic_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2500 2501class MUBUF_Real_Atomic_gfx12_impl<bits<8> op, string ps_name, 2502 string real_name> : 2503 VBUFFER_MUBUF_Real_gfx12<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2504 2505multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return, 2506 string real_name> { 2507 let DecoderNamespace = "GFX11" in { 2508 defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); 2509 def _BOTHEN#Rtn#_gfx11 : 2510 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_BOTHEN" # Rtn, real_name>, 2511 AtomicNoRet<NAME # "_BOTHEN_gfx11", is_return>; 2512 def _IDXEN#Rtn#_gfx11 : 2513 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_IDXEN" # Rtn, real_name>, 2514 AtomicNoRet<NAME # "_IDXEN_gfx11", is_return>; 2515 def _OFFEN#Rtn#_gfx11 : 2516 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFEN" # Rtn, real_name>, 2517 AtomicNoRet<NAME # "_OFFEN_gfx11", is_return>; 2518 def _OFFSET#Rtn#_gfx11 : 2519 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFSET" # Rtn, real_name>, 2520 AtomicNoRet<NAME # "_OFFSET_gfx11", is_return>; 2521 } 2522} 2523 2524multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl<bits<8> op, bit is_return, 2525 string real_name> { 2526 let DecoderNamespace = "GFX12" in { 2527 defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); 2528 def _BOTHEN#Rtn#_gfx12 : 2529 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN" # Rtn, real_name>, 2530 AtomicNoRet<NAME # "_BOTHEN_gfx12", is_return>; 2531 def _IDXEN#Rtn#_gfx12 : 2532 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_IDXEN" # Rtn, real_name>, 2533 AtomicNoRet<NAME # "_IDXEN_gfx12", is_return>; 2534 def _OFFEN#Rtn#_gfx12 : 2535 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_OFFEN" # Rtn, real_name>, 2536 AtomicNoRet<NAME # "_OFFEN_gfx12", is_return>; 2537 def _OFFSET#Rtn#_gfx12 : 2538 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_OFFSET" # Rtn, real_name>, 2539 AtomicNoRet<NAME # "_OFFSET_gfx12", is_return>; 2540 } 2541} 2542 2543multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<bits<8> op, bit is_return, 2544 string real_name> : 2545 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, is_return, real_name>, 2546 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, is_return, real_name>; 2547 2548// Non-renamed gfx11/gfx12 mubuf atomic. 2549multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op> : 2550 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>, 2551 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>; 2552 2553multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> : 2554 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>, 2555 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>; 2556 2557multiclass MUBUF_Real_Atomic_gfx11_Renamed<bits<8> op, string real_name> : 2558 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, real_name>, 2559 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, real_name> { 2560 def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2561} 2562 2563multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed<bits<8> op, string real_name> : 2564 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 0, real_name>, 2565 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 1, real_name> { 2566 def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2567} 2568 2569multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<bits<8> op, string gfx12_name, string gfx11_name> : 2570 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, gfx11_name>, 2571 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, gfx11_name>, 2572 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 0, gfx12_name>, 2573 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 1, gfx12_name> { 2574 def : Mnem_gfx11<get_MUBUF_ps<NAME>.Mnemonic, gfx11_name>; 2575 def : Mnem_gfx12<get_MUBUF_ps<NAME>.Mnemonic, gfx12_name>; 2576 def : Mnem_gfx12<gfx11_name, gfx12_name>; 2577} 2578 2579let DecoderNamespace = "GFX11" in { 2580def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2581def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2582} 2583 2584defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x014, "buffer_load_b32">; 2585defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x015, "buffer_load_b64">; 2586defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x016, "buffer_load_b96">; 2587defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x017, "buffer_load_b128">; 2588defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x020, "buffer_load_d16_b16">; 2589defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "buffer_load_d16_format_x">; 2590defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "buffer_load_d16_format_xy">; 2591defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2592defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2593defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x023, "buffer_load_d16_hi_b16">; 2594defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2595defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x022, "buffer_load_d16_hi_i8">; 2596defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x021, "buffer_load_d16_hi_u8">; 2597defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01f, "buffer_load_d16_i8">; 2598defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01e, "buffer_load_d16_u8">; 2599defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2600defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2601defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2602defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2603defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x011, "buffer_load_i8">; 2604defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x013, "buffer_load_i16">; 2605defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x010, "buffer_load_u8">; 2606defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x012, "buffer_load_u16">; 2607defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>; 2608defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>; 2609defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>; 2610defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>; 2611defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>; 2612defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>; 2613defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x018, "buffer_store_b8">; 2614defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x019, "buffer_store_b16">; 2615defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01A, "buffer_store_b32">; 2616defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01B, "buffer_store_b64">; 2617defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01C, "buffer_store_b96">; 2618defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01D, "buffer_store_b128">; 2619defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00C, "buffer_store_d16_format_x">; 2620defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00D, "buffer_store_d16_format_xy">; 2621defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2622defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2623defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x024, "buffer_store_d16_hi_b8">; 2624defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x025, "buffer_store_d16_hi_b16">; 2625defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2626defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2627defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2628defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2629defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2630defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11_gfx12<0x056>; 2631defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x035, "buffer_atomic_add_u32">; 2632defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x043, "buffer_atomic_add_u64">; 2633defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03C, "buffer_atomic_and_b32">; 2634defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x049, "buffer_atomic_and_b64">; 2635defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2636defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2637defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2638defm BUFFER_ATOMIC_COND_SUB_U32 : MUBUF_Real_Atomic_gfx12<0x050>; 2639defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">; 2640def : Mnem_gfx11_gfx12<"buffer_atomic_csub", "buffer_atomic_csub_u32">; 2641defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x040, "buffer_atomic_dec_u32">; 2642defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04D, "buffer_atomic_dec_u64">; 2643defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03F, "buffer_atomic_inc_u32">; 2644defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04C, "buffer_atomic_inc_u64">; 2645defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">; 2646defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03A, "buffer_atomic_max_i32">; 2647defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x047, "buffer_atomic_max_i64">; 2648defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03B, "buffer_atomic_max_u32">; 2649defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x048, "buffer_atomic_max_u64">; 2650defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">; 2651defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x038, "buffer_atomic_min_i32">; 2652defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x045, "buffer_atomic_min_i64">; 2653defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x039, "buffer_atomic_min_u32">; 2654defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x046, "buffer_atomic_min_u64">; 2655defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03D, "buffer_atomic_or_b32">; 2656defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04A, "buffer_atomic_or_b64">; 2657defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x036, "buffer_atomic_sub_u32">; 2658defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x044, "buffer_atomic_sub_u64">; 2659defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x033, "buffer_atomic_swap_b32">; 2660defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x041, "buffer_atomic_swap_b64">; 2661defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03E, "buffer_atomic_xor_b32">; 2662defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04B, "buffer_atomic_xor_b64">; 2663defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_gfx12<0x059>; 2664defm BUFFER_ATOMIC_PK_ADD_BF16 : MUBUF_Real_Atomic_gfx12<0x05a>; 2665 2666//===----------------------------------------------------------------------===// 2667// MUBUF - GFX10. 2668//===----------------------------------------------------------------------===// 2669 2670let DecoderNamespace = "GFX10" in { 2671 multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> { 2672 def _BOTHEN_gfx10 : 2673 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2674 def _IDXEN_gfx10 : 2675 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2676 def _OFFEN_gfx10 : 2677 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2678 def _OFFSET_gfx10 : 2679 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2680 } 2681 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2682 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2683 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2684 } 2685 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> { 2686 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2687 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2688 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2689 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2690 2691 if !not(isTFE) then { 2692 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2693 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2694 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2695 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2696 } 2697 } 2698 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2699 def _BOTHEN_RTN_gfx10 : 2700 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2701 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2702 def _IDXEN_RTN_gfx10 : 2703 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2704 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2705 def _OFFEN_RTN_gfx10 : 2706 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2707 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2708 def _OFFSET_RTN_gfx10 : 2709 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2710 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2711 } 2712 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2713 MUBUF_Real_Atomics_RTN_gfx10<op> { 2714 def _BOTHEN_gfx10 : 2715 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2716 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2717 def _IDXEN_gfx10 : 2718 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2719 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2720 def _OFFEN_gfx10 : 2721 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2722 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2723 def _OFFSET_gfx10 : 2724 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2725 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2726 } 2727} // End DecoderNamespace = "GFX10" 2728 2729defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2730defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2731defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2732defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2733defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2734defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2735defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2736defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2737// FIXME-GFX10: Add following instructions: 2738//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2739//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2740defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2741defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2742defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2743defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2744defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2745defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2746defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2747defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2748 2749def BUFFER_GL0_INV_gfx10 : 2750 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2751def BUFFER_GL1_INV_gfx10 : 2752 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2753 2754//===----------------------------------------------------------------------===// 2755// MUBUF - GFX6, GFX7, GFX10. 2756//===----------------------------------------------------------------------===// 2757 2758let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2759 multiclass MUBUF_Real_gfx6<bits<8> op> { 2760 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2761 } 2762} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2763 2764let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2765 multiclass MUBUF_Real_gfx7<bits<8> op> { 2766 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2767 } 2768} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2769 2770let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2771 multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> { 2772 def _ADDR64_gfx6_gfx7 : 2773 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2774 def _BOTHEN_gfx6_gfx7 : 2775 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2776 def _IDXEN_gfx6_gfx7 : 2777 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2778 def _OFFEN_gfx6_gfx7 : 2779 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2780 def _OFFSET_gfx6_gfx7 : 2781 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2782 } 2783 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2784 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2785 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2786 } 2787 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> { 2788 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2789 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2790 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2791 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2792 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2793 2794 if !not(isTFE) then { 2795 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2796 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2797 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2798 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2799 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2800 } 2801 } 2802 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2803 def _ADDR64_gfx6_gfx7 : 2804 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2805 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2806 def _BOTHEN_gfx6_gfx7 : 2807 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2808 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2809 def _IDXEN_gfx6_gfx7 : 2810 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2811 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2812 def _OFFEN_gfx6_gfx7 : 2813 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2814 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2815 def _OFFSET_gfx6_gfx7 : 2816 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2817 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2818 2819 def _ADDR64_RTN_gfx6_gfx7 : 2820 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2821 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2822 def _BOTHEN_RTN_gfx6_gfx7 : 2823 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2824 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2825 def _IDXEN_RTN_gfx6_gfx7 : 2826 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2827 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2828 def _OFFEN_RTN_gfx6_gfx7 : 2829 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2830 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2831 def _OFFSET_RTN_gfx6_gfx7 : 2832 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2833 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2834 } 2835} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2836 2837multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2838 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2839 2840multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> : 2841 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>, 2842 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>; 2843 2844multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> { 2845 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>; 2846 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>; 2847} 2848 2849multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2850 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2851 2852// FIXME-GFX6: Following instructions are available only on GFX6. 2853//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2854//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2855 2856defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2857defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2858defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2859defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2860defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2861defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2862defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2863defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2864defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2865defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2866defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2867defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2868defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2869defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2870defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2871defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2872defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2873defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2874defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2875defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2876defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2877defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2878 2879defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2880defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2881defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2882defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2883defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2884defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2885defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2886defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2887defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2888defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2889defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2890defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2891defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2892defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2893defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2894defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2895defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2896defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2897defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2898defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2899defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2900defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2901defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2902defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2903defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2904defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2905defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2906defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2907defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2908// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2909defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2910defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2911defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2912 2913defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>; 2914 2915defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2916defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2917def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2918 2919//===----------------------------------------------------------------------===// 2920// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2921//===----------------------------------------------------------------------===// 2922 2923class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2924 string real_name = ps.Mnemonic> : 2925 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2926 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2927 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2928 let Inst{31-26} = 0x3a; //encoding 2929 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2930 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2931 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2932 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2933} 2934 2935class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2936 string real_name = ps.Mnemonic> : 2937 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2938 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2939 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2940 let Inst{18-15} = op; 2941 let Inst{25-19} = format; 2942 let Inst{53} = ps.tfe; 2943 let Inst{54} = ps.offen; 2944 let Inst{55} = ps.idxen; 2945} 2946 2947class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2948 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2949 let Inst{12} = ps.offen; 2950 let Inst{13} = ps.idxen; 2951 let Inst{18-16} = op; 2952 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2953 let Inst{55} = ps.tfe; 2954} 2955 2956//===----------------------------------------------------------------------===// 2957// MTBUF - GFX11. 2958//===----------------------------------------------------------------------===// 2959 2960multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<bits<4> op, string real_name> { 2961 let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2962 def _BOTHEN_gfx11 : 2963 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2964 def _IDXEN_gfx11 : 2965 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2966 def _OFFEN_gfx11 : 2967 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2968 def _OFFSET_gfx11 : 2969 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2970 } 2971 2972 let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in { 2973 def _BOTHEN_gfx12 : 2974 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_BOTHEN"), real_name>; 2975 def _IDXEN_gfx12 : 2976 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_IDXEN"), real_name>; 2977 def _OFFEN_gfx12 : 2978 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_OFFEN"), real_name>; 2979 def _OFFSET_gfx12 : 2980 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_OFFSET"), real_name>; 2981 } 2982} 2983 2984multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, MTBUF_Pseudo ps> 2985 : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, ps.Mnemonic>; 2986multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op> 2987 : MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2988 2989 2990class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2991 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2992multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<bits<4> op, string real_name> 2993 : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, real_name> { 2994 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2995} 2996 2997defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">; 2998defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2999defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 3000defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 3001defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>; 3002defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>; 3003defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>; 3004defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>; 3005defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00c, "tbuffer_store_d16_format_x">; 3006defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 3007defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 3008defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 3009defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>; 3010defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>; 3011defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>; 3012defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>; 3013 3014//===----------------------------------------------------------------------===// 3015// MTBUF - GFX10. 3016//===----------------------------------------------------------------------===// 3017 3018class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 3019 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 3020 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 3021 let Inst{25-19} = format; 3022 let Inst{53} = op{3}; 3023} 3024 3025let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 3026 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 3027 def _BOTHEN_gfx10 : 3028 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3029 def _IDXEN_gfx10 : 3030 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3031 def _OFFEN_gfx10 : 3032 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3033 def _OFFSET_gfx10 : 3034 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3035 } 3036} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 3037 3038defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 3039defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 3040defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 3041defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 3042defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 3043defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 3044defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 3045defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 3046 3047//===----------------------------------------------------------------------===// 3048// MTBUF - GFX6, GFX7, GFX10. 3049//===----------------------------------------------------------------------===// 3050 3051class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 3052 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 3053 let Inst{15} = ps.addr64; 3054 let Inst{22-19} = dfmt; 3055 let Inst{25-23} = nfmt; 3056} 3057 3058let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 3059 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 3060 def _ADDR64_gfx6_gfx7 : 3061 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 3062 def _BOTHEN_gfx6_gfx7 : 3063 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3064 def _IDXEN_gfx6_gfx7 : 3065 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3066 def _OFFEN_gfx6_gfx7 : 3067 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3068 def _OFFSET_gfx6_gfx7 : 3069 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3070 } 3071} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 3072 3073multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 3074 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 3075 3076defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 3077defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 3078defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 3079defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 3080defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 3081defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 3082defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 3083defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 3084 3085//===----------------------------------------------------------------------===// 3086// GFX8, GFX9 (VI). 3087//===----------------------------------------------------------------------===// 3088 3089class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 3090 bit has_sccb = ps.has_sccb> : 3091 MUBUF_Real<ps>, 3092 Enc64, 3093 SIMCInstr<ps.PseudoInstr, Enc>, 3094 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 3095 !if(ps.IsAtomicRet, 1, ?))> { 3096 3097 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3098 let Inst{12} = ps.offen; 3099 let Inst{13} = ps.idxen; 3100 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3101 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3102 let Inst{16} = ps.lds; 3103 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3104 let Inst{24-18} = op; 3105 let Inst{31-26} = 0x38; //encoding 3106 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3107 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3108 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3109 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3110} 3111 3112class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 3113 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 3114 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3115 let DecoderNamespace = "GFX8"; 3116 3117 let Inst{55} = ps.tfe; 3118} 3119 3120class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 3121 bit has_sccb = ps.has_sccb> : 3122 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 3123 let AssemblerPredicate = isGFX90APlus; 3124 let DecoderNamespace = "GFX90A"; 3125 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 3126 ps.AsmOperands); 3127 3128 let Inst{55} = acc; 3129} 3130 3131class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 3132 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 3133 let AssemblerPredicate = isGFX940Plus; 3134 let DecoderNamespace = "GFX9"; 3135 let AsmString = ps.Mnemonic # ps.AsmOperands; 3136 3137 let Inst{55} = acc; 3138} 3139 3140multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps, bit isTFE = 0> { 3141 def _vi : MUBUF_Real_vi<op, ps>; 3142 3143 if !not(isTFE) then { 3144 if !not(ps.FPAtomic) then 3145 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 3146 } 3147 3148 if ps.FPAtomic then { 3149 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 3150 let SubtargetPredicate = isGFX90AOnly; 3151 let AssemblerPredicate = isGFX90AOnly; 3152 } 3153 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 3154 } 3155} 3156 3157multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> { 3158 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), isTFE>; 3159 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), isTFE>; 3160 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), isTFE>; 3161 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), isTFE>; 3162} 3163 3164multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> { 3165 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>; 3166 if hasTFE then 3167 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>; 3168} 3169 3170multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> { 3171 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 3172 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 3173 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 3174 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 3175 3176 if !not(isTFE) then { 3177 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 3178 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 3179 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 3180 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 3181 3182 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 3183 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 3184 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 3185 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 3186 3187 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 3188 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 3189 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 3190 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 3191 } 3192} 3193 3194multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 3195 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>; 3196 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>; 3197} 3198 3199class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 3200 MUBUF_Real<ps>, 3201 Enc64, 3202 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3203 let AssemblerPredicate=HasUnpackedD16VMem; 3204 let DecoderNamespace="GFX80_UNPACKED"; 3205 3206 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3207 let Inst{12} = ps.offen; 3208 let Inst{13} = ps.idxen; 3209 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3210 let Inst{16} = ps.lds; 3211 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3212 let Inst{24-18} = op; 3213 let Inst{31-26} = 0x38; //encoding 3214 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3215 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3216 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3217 let Inst{55} = ps.tfe; 3218 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3219} 3220 3221multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> { 3222 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 3223 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 3224 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 3225 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 3226} 3227 3228multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 3229 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3230 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3231} 3232 3233multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 3234 MUBUF_Real_AllAddr_vi<op, 0> { 3235 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 3236 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 3237 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 3238 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 3239} 3240 3241defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 3242defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 3243defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 3244defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 3245defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 3246defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 3247defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 3248defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 3249let SubtargetPredicate = HasUnpackedD16VMem in { 3250 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 3251 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 3252 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 3253 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 3254 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 3255 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 3256 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 3257 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 3258} // End HasUnpackedD16VMem. 3259let SubtargetPredicate = HasPackedD16VMem in { 3260 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 3261 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 3262 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 3263 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 3264 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 3265 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 3266 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 3267 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 3268} // End HasPackedD16VMem. 3269defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 3270defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 3271defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 3272defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 3273defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 3274defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 3275defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 3276defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 3277defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 3278defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 3279defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 3280defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 3281defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 3282defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 3283defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 3284defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 3285 3286defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 3287defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 3288defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 3289defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 3290defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 3291defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 3292 3293defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 3294defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 3295 3296defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 3297defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 3298defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 3299defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 3300defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 3301defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 3302defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 3303defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 3304defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 3305defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 3306defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 3307defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 3308defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 3309 3310defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 3311defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 3312defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 3313defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 3314defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 3315defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 3316defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 3317defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 3318defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 3319defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 3320defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 3321defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 3322defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 3323 3324defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 3325 3326let AssemblerPredicate = isGFX8GFX9 in { 3327def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 3328def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 3329} // End AssemblerPredicate = isGFX8GFX9 3330 3331 3332defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 3333 3334let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 3335defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 3336} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 3337 3338let SubtargetPredicate = isGFX90APlus in { 3339 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 3340 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 3341 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 3342} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 3343 3344def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 3345 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 3346 let AssemblerPredicate = isGFX90AOnly; 3347 let SubtargetPredicate = isGFX90AOnly; 3348} 3349def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 3350 3351let SubtargetPredicate = isGFX940Plus in { 3352def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 3353def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 3354} 3355 3356class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 3357 MTBUF_Real<ps>, 3358 Enc64, 3359 SIMCInstr<ps.PseudoInstr, Enc> { 3360 3361 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3362 let Inst{12} = ps.offen; 3363 let Inst{13} = ps.idxen; 3364 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3365 let Inst{18-15} = op; 3366 let Inst{22-19} = dfmt; 3367 let Inst{25-23} = nfmt; 3368 let Inst{31-26} = 0x3a; //encoding 3369 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3370 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3371 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3372 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3373 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3374 let Inst{55} = ps.tfe; 3375 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3376} 3377 3378class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 3379 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 3380 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3381 let DecoderNamespace = "GFX8"; 3382 3383 let Inst{55} = ps.tfe; 3384} 3385 3386class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 3387 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 3388 let AssemblerPredicate = isGFX90APlus; 3389 let DecoderNamespace = "GFX90A"; 3390 let AsmString = ps.Mnemonic # ps.AsmOperands; 3391 3392 let Inst{55} = acc; 3393} 3394 3395multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 3396 def _vi : MTBUF_Real_vi<op, ps>; 3397 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 3398} 3399 3400multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 3401 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3402 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3403 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3404 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3405} 3406 3407class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 3408 MTBUF_Real<ps>, 3409 Enc64, 3410 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3411 let AssemblerPredicate=HasUnpackedD16VMem; 3412 let DecoderNamespace="GFX80_UNPACKED"; 3413 3414 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3415 let Inst{12} = ps.offen; 3416 let Inst{13} = ps.idxen; 3417 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3418 let Inst{18-15} = op; 3419 let Inst{22-19} = dfmt; 3420 let Inst{25-23} = nfmt; 3421 let Inst{31-26} = 0x3a; //encoding 3422 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3423 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3424 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3425 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3426 let Inst{55} = ps.tfe; 3427 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3428} 3429 3430multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 3431 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3432 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3433 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3434 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3435} 3436 3437defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 3438defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 3439defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 3440defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 3441defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 3442defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 3443defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 3444defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 3445let SubtargetPredicate = HasUnpackedD16VMem in { 3446 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 3447 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 3448 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 3449 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 3450 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 3451 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 3452 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 3453 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 3454} // End HasUnpackedD16VMem. 3455let SubtargetPredicate = HasPackedD16VMem in { 3456 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 3457 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 3458 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 3459 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 3460 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 3461 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 3462 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 3463 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 3464} // End HasUnpackedD16VMem. 3465 3466def MUBUFInfoTable : GenericTable { 3467 let FilterClass = "MUBUF_Pseudo"; 3468 let CppTypeName = "MUBUFInfo"; 3469 let Fields = [ 3470 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3471 "IsBufferInv" 3472 ]; 3473 3474 let PrimaryKey = ["Opcode"]; 3475 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3476} 3477 3478def getMUBUFInfoFromOpcode : SearchIndex { 3479 let Table = MUBUFInfoTable; 3480 let Key = ["Opcode"]; 3481} 3482 3483def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3484 let Table = MUBUFInfoTable; 3485 let Key = ["BaseOpcode", "elements"]; 3486} 3487 3488def MTBUFInfoTable : GenericTable { 3489 let FilterClass = "MTBUF_Pseudo"; 3490 let CppTypeName = "MTBUFInfo"; 3491 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3492 3493 let PrimaryKey = ["Opcode"]; 3494 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3495} 3496 3497def getMTBUFInfoFromOpcode : SearchIndex { 3498 let Table = MTBUFInfoTable; 3499 let Key = ["Opcode"]; 3500} 3501 3502def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3503 let Table = MTBUFInfoTable; 3504 let Key = ["BaseOpcode", "elements"]; 3505} 3506