1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFSOffset : ComplexPattern<iPTR, 1, "SelectBUFSOffset">; 16 17def BUFAddrKind { 18 int Offset = 0; 19 int OffEn = 1; 20 int IdxEn = 2; 21 int BothEn = 3; 22 int Addr64 = 4; 23} 24 25class getAddrName<int addrKind> { 26 string ret = 27 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 28 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 29 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 30 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 31 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 32 ""))))); 33} 34 35class MUBUFAddr64Table <bit is_addr64, string Name> { 36 bit IsAddr64 = is_addr64; 37 string OpName = Name; 38} 39 40class MTBUFAddr64Table <bit is_addr64, string Name> { 41 bit IsAddr64 = is_addr64; 42 string OpName = Name; 43} 44 45 46//===----------------------------------------------------------------------===// 47// BUF class (base class for MTBUF and MUBUF pseudos) 48//===----------------------------------------------------------------------===// 49 50class BUF_Pseudo <string opName, dag outs, dag ins, 51 string asmOps, list<dag> pattern=[]> : 52 InstSI<outs, ins, "", pattern>, 53 SIMCInstr<opName, SIEncodingFamily.NONE> { 54 55 let isPseudo = 1; 56 let isCodeGenOnly = 1; 57 let Size = 8; 58 let UseNamedOperandTable = 1; 59 60 string Mnemonic = opName; 61 string AsmOperands = asmOps; 62 63 Instruction Opcode = !cast<Instruction>(NAME); 64 65 66 let VM_CNT = 1; 67 let EXP_CNT = 1; 68 69 let Uses = [EXEC]; 70 let hasSideEffects = 0; 71 let SchedRW = [WriteVMEM]; 72 73 74 75 bits<1> offen = 0; 76 bits<1> idxen = 0; 77 bits<1> addr64 = 0; 78 bits<1> lds = 0; 79 bits<1> has_vdata = !not(lds); 80 bits<1> has_vaddr = 1; 81 bits<1> has_glc = 1; 82 bits<1> has_dlc = 1; 83 bits<1> glc_value = 0; // the value for glc if no such operand 84 bits<1> dlc_value = 0; // the value for dlc if no such operand 85 bits<1> has_srsrc = 1; 86 bits<1> has_soffset = 1; 87 bits<1> has_offset = 1; 88 bits<1> has_slc = 1; 89 bits<1> tfe = ?; 90 bits<4> elements = 0; 91 bits<1> has_sccb = 1; 92 bits<1> sccb_value = 0; 93 bits<1> IsBufferInv = 0; 94} 95 96 97 98//===----------------------------------------------------------------------===// 99// MTBUF classes 100//===----------------------------------------------------------------------===// 101 102class MTBUFGetBaseOpcode<string Op> { 103 string ret = !subst("FORMAT_XY", "FORMAT_X", 104 !subst("FORMAT_XYZ", "FORMAT_X", 105 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 106} 107 108 109class MTBUF_Pseudo <string opName, dag outs, dag ins, 110 string asmOps, list<dag> pattern=[]> : 111 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 112 113 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 114 let MTBUF = 1; 115} 116 117class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 118 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 119 120 let isPseudo = 0; 121 let isCodeGenOnly = 0; 122 123 let VM_CNT = 1; 124 let EXP_CNT = 1; 125 let MTBUF = 1; 126 127 // copy relevant pseudo op flags 128 let UseNamedOperandTable = ps.UseNamedOperandTable; 129 let SubtargetPredicate = ps.SubtargetPredicate; 130 let AsmMatchConverter = ps.AsmMatchConverter; 131 let Constraints = ps.Constraints; 132 let DisableEncoding = ps.DisableEncoding; 133 let TSFlags = ps.TSFlags; 134 let SchedRW = ps.SchedRW; 135 let mayLoad = ps.mayLoad; 136 let mayStore = ps.mayStore; 137 let IsAtomicRet = ps.IsAtomicRet; 138 let IsAtomicNoRet = ps.IsAtomicNoRet; 139 140 bits<12> offset; 141 bits<5> cpol; 142 bits<7> format; 143 bits<8> vaddr; 144 bits<10> vdata; 145 bits<7> srsrc; 146 bits<8> soffset; 147 148 bits<4> dfmt = format{3-0}; 149 bits<3> nfmt = format{6-4}; 150 151 // GFX90A+ only: instruction uses AccVGPR for data 152 // Bit supersedes tfe. 153 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 154} 155 156class getMTBUFInsDA<list<RegisterClass> vdataList, 157 list<RegisterClass> vaddrList=[], bit hasGFX12Enc> { 158 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 159 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 160 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 161 162 dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), 163 (ins SCSrc_b32:$soffset)); 164 165 dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, 166 (ins offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz)); 167 168 dag Inputs = !if(!empty(vaddrList), 169 NonVaddrInputs, 170 !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 171 dag ret = !if(!empty(vdataList), 172 Inputs, 173 !con((ins vdata_op:$vdata), Inputs)); 174} 175 176class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit hasGFX12Enc> { 177 dag ret = 178 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasGFX12Enc>.ret, 179 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32], hasGFX12Enc>.ret, 180 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32], hasGFX12Enc>.ret, 181 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasGFX12Enc>.ret, 182 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasGFX12Enc>.ret, 183 (ins)))))); 184} 185 186class getMTBUFAsmOps<int addrKind> { 187 string Pfx = 188 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 189 !if(!eq(addrKind, BUFAddrKind.OffEn), 190 "$vaddr, $srsrc,$format $soffset offen", 191 !if(!eq(addrKind, BUFAddrKind.IdxEn), 192 "$vaddr, $srsrc,$format $soffset idxen", 193 !if(!eq(addrKind, BUFAddrKind.BothEn), 194 "$vaddr, $srsrc,$format $soffset idxen offen", 195 !if(!eq(addrKind, BUFAddrKind.Addr64), 196 "$vaddr, $srsrc,$format $soffset addr64", 197 ""))))); 198 string ret = " $vdata, " # Pfx # "$offset$cpol"; 199} 200 201class MTBUF_SetupAddr<int addrKind> { 202 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 203 !eq(addrKind, BUFAddrKind.BothEn)); 204 205 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 206 !eq(addrKind, BUFAddrKind.BothEn)); 207 208 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 209 210 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 211} 212 213class MTBUF_Load_Pseudo <string opName, 214 int addrKind, 215 RegisterClass vdataClass, 216 int elems, 217 bit hasGFX12Enc = 0, 218 list<dag> pattern=[], 219 // Workaround bug bz30254 220 int addrKindCopy = addrKind> 221 : MTBUF_Pseudo<opName, 222 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 223 getMTBUFIns<addrKindCopy, [], hasGFX12Enc>.ret, 224 getMTBUFAsmOps<addrKindCopy>.ret, 225 pattern>, 226 MTBUF_SetupAddr<addrKindCopy> { 227 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 228 let mayLoad = 1; 229 let mayStore = 0; 230 let elements = elems; 231} 232 233multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterClass vdataClass, 234 int elems, bit hasGFX12Enc> { 235 236 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>, 237 MTBUFAddr64Table<0, NAME>; 238 239 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasGFX12Enc>, 240 MTBUFAddr64Table<1, NAME>; 241 242 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 243 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 244 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 245 246 let DisableWQM = 1 in { 247 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>; 248 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 249 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 250 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 251 } 252} 253 254multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 255 int elems> { 256 defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>; 257 defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>; 258} 259 260class MTBUF_Store_Pseudo <string opName, 261 int addrKind, 262 RegisterClass vdataClass, 263 int elems, 264 bit hasGFX12Enc = 0, 265 list<dag> pattern=[], 266 // Workaround bug bz30254 267 int addrKindCopy = addrKind, 268 RegisterClass vdataClassCopy = vdataClass> 269 : MTBUF_Pseudo<opName, 270 (outs), 271 getMTBUFIns<addrKindCopy, [vdataClassCopy], hasGFX12Enc>.ret, 272 getMTBUFAsmOps<addrKindCopy>.ret, 273 pattern>, 274 MTBUF_SetupAddr<addrKindCopy> { 275 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 276 let mayLoad = 0; 277 let mayStore = 1; 278 let elements = elems; 279} 280 281multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterClass vdataClass, 282 int elems, bit hasGFX12Enc> { 283 284 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>, 285 MTBUFAddr64Table<0, NAME>; 286 287 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasGFX12Enc>, 288 MTBUFAddr64Table<1, NAME>; 289 290 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 291 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 292 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 293 294 let DisableWQM = 1 in { 295 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>; 296 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>; 297 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>; 298 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>; 299 } 300} 301 302multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 303 int elems> { 304 defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>; 305 defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>; 306} 307 308//===----------------------------------------------------------------------===// 309// MUBUF classes 310//===----------------------------------------------------------------------===// 311 312class MUBUFGetBaseOpcode<string Op> { 313 string ret = !subst("DWORDX2", "DWORD", 314 !subst("DWORDX3", "DWORD", 315 !subst("DWORDX4", "DWORD", Op))); 316} 317 318class MUBUF_Pseudo <string opName, dag outs, dag ins, 319 string asmOps, list<dag> pattern=[]> : 320 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 321 322 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 323 let MUBUF = 1; 324 let AsmMatchConverter = "cvtMubuf"; 325} 326 327class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 328 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 329 330 let isPseudo = 0; 331 let isCodeGenOnly = 0; 332 333 let VM_CNT = 1; 334 let EXP_CNT = 1; 335 let MUBUF = 1; 336 337 // copy relevant pseudo op flags 338 let SubtargetPredicate = ps.SubtargetPredicate; 339 let AsmMatchConverter = ps.AsmMatchConverter; 340 let OtherPredicates = ps.OtherPredicates; 341 let Constraints = ps.Constraints; 342 let DisableEncoding = ps.DisableEncoding; 343 let TSFlags = ps.TSFlags; 344 let UseNamedOperandTable = ps.UseNamedOperandTable; 345 let SchedRW = ps.SchedRW; 346 let mayLoad = ps.mayLoad; 347 let mayStore = ps.mayStore; 348 let IsAtomicRet = ps.IsAtomicRet; 349 let IsAtomicNoRet = ps.IsAtomicNoRet; 350 let VALU = ps.VALU; 351 let LGKM_CNT = ps.LGKM_CNT; 352 353 bits<12> offset; 354 bits<5> cpol; 355 bits<8> vaddr; 356 bits<10> vdata; 357 bits<7> srsrc; 358 bits<8> soffset; 359 360 // GFX90A+ only: instruction uses AccVGPR for data 361 // Bit supersedes tfe. 362 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 363} 364 365// For cache invalidation instructions. 366class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 367 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 368 369 let AsmMatchConverter = ""; 370 371 let hasSideEffects = 1; 372 let mayLoad = 0; 373 let mayStore = 0; 374 375 let IsBufferInv = 1; 376 // Set everything else to 0. 377 let offen = 0; 378 let idxen = 0; 379 let addr64 = 0; 380 let has_vdata = 0; 381 let has_vaddr = 0; 382 let has_glc = 0; 383 let has_dlc = 0; 384 let glc_value = 0; 385 let dlc_value = 0; 386 let has_srsrc = 0; 387 let has_soffset = 0; 388 let has_offset = 0; 389 let has_slc = 0; 390 let has_sccb = 0; 391 let sccb_value = 0; 392} 393 394class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> { 395 RegisterOperand tfeVDataOp = 396 !if(!eq(RC.Size, 32), AVLdSt_64, 397 !if(!eq(RC.Size, 64), AVLdSt_96, 398 !if(!eq(RC.Size, 96), AVLdSt_128, 399 !if(!eq(RC.Size, 128), AVLdSt_160, 400 RegisterOperand<VReg_1> // Invalid register. 401 )))); 402 403 RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret); 404} 405 406class getMUBUFInsDA<list<RegisterClass> vdataList, 407 list<RegisterClass> vaddrList, bit isTFE, bit hasGFX12Enc> { 408 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 409 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 410 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret; 411 412 dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 413 dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, (ins offset:$offset, CPol_0:$cpol, i1imm_0:$swz)); 414 415 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 416 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 417} 418 419class getMUBUFElements<ValueType vt> { 420 int ret = 421 !if(!eq(vt, f16), 1, 422 !if(!eq(vt, v2f16), 2, 423 !if(!eq(vt, v3f16), 3, 424 !if(!eq(vt, v4f16), 4, 425 !if(!eq(vt.Size, 32), 1, 426 !if(!eq(vt.Size, 64), 2, 427 !if(!eq(vt.Size, 96), 3, 428 !if(!eq(vt.Size, 128), 4, 0) 429 ) 430 ) 431 ) 432 ) 433 ) 434 ) 435 ); 436} 437 438class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE, bit hasGFX12Enc> { 439 dag ret = 440 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasGFX12Enc>.ret, 441 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasGFX12Enc>.ret, 442 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasGFX12Enc>.ret, 443 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasGFX12Enc>.ret, 444 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasGFX12Enc>.ret, 445 (ins)))))); 446} 447 448class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> { 449 string Vdata = !if(noVdata, " ", " $vdata, "); 450 string Lds = !if(isLds, " lds", ""); 451 string TFE = !if(isTFE, " tfe", ""); 452 string MainArgs = 453 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 454 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 455 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 456 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 457 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 458 ""))))); 459 string Offset = "$offset"; 460 string OtherArgs = "$cpol"; 461 462 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE; 463} 464 465class MUBUF_SetupAddr<int addrKind> { 466 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 467 !eq(addrKind, BUFAddrKind.BothEn)); 468 469 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 470 !eq(addrKind, BUFAddrKind.BothEn)); 471 472 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 473 474 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 475} 476 477class MUBUF_Load_Pseudo <string opName, 478 int addrKind, 479 ValueType vdata_vt, 480 bit HasTiedDest = 0, 481 bit isLds = 0, 482 bit isLdsOpc = 0, 483 bit isTFE = 0, 484 bit hasGFX12Enc = 0, 485 list<dag> pattern=[], 486 // Workaround bug bz30254 487 int addrKindCopy = addrKind, 488 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 489 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret> 490 : MUBUF_Pseudo<opName, 491 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 492 !con(getMUBUFIns<addrKindCopy, [], isTFE, hasGFX12Enc>.ret, 493 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 494 getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret, 495 pattern>, 496 MUBUF_SetupAddr<addrKindCopy> { 497 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") # 498 "_" # getAddrName<addrKindCopy>.ret; 499 let AsmMatchConverter = "cvtMubuf"; 500 501 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 502 let LGKM_CNT = isLds; 503 let has_vdata = !not(!or(isLds, isLdsOpc)); 504 let mayLoad = 1; 505 let mayStore = isLds; 506 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 507 let tfe = isTFE; 508 let lds = isLds; 509 let elements = getMUBUFElements<vdata_vt>.ret; 510 let VALU = isLds; 511} 512 513class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : GCNPat < 514 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 515 (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset)) 516>; 517 518class MUBUF_Addr64_Load_Pat <Instruction inst, 519 ValueType load_vt = i32, 520 SDPatternOperator ld = null_frag> : GCNPat < 521 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 522 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)) 523>; 524 525multiclass MUBUF_Pseudo_Load_Pats_Common<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 526 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 527 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 528} 529 530multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag>{ 531 let SubtargetPredicate = HasUnrestrictedSOffset in { 532 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst, load_vt, ld>; 533 } 534 defm : MUBUF_Pseudo_Load_Pats_Common<BaseInst # "_VBUFFER", load_vt, ld>; 535} 536 537multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt, 538 bit TiedDest, bit isLds, bit isTFE, bit hasGFX12Enc> { 539 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 540 541 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>, 542 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 543 544 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>, 545 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 546 547 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 548 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 549 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 550 551 let DisableWQM = 1 in { 552 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 553 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 554 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 555 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>; 556 } 557} 558 559multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32, 560 bit TiedDest = 0, bit isLds = 0> { 561 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>; 562 defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>; 563 564 if !not(isLds) then { 565 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>; 566 defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>; 567 } 568} 569 570multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 571 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 572 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 573} 574 575multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 576 ValueType load_vt = i32, 577 bit TiedDest = 0, 578 bit isLds = 0, 579 bit isLdsOpc = 1> { 580 581 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 582 583 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 584 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 585 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 586 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 587 588 def _VBUFFER_OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 589 def _VBUFFER_OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 590 def _VBUFFER_IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 591 def _VBUFFER_BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>; 592} 593 594class MUBUF_Store_Pseudo <string opName, 595 int addrKind, 596 ValueType store_vt, 597 bit isTFE = 0, 598 bit hasGFX12Enc = 0, 599 list<dag> pattern=[], 600 // Workaround bug bz30254 601 int addrKindCopy = addrKind> 602 : MUBUF_Pseudo<opName, 603 (outs), 604 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret], isTFE, hasGFX12Enc>.ret, 605 getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret, 606 pattern>, 607 MUBUF_SetupAddr<addrKindCopy> { 608 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") # 609 getAddrName<addrKindCopy>.ret; 610 let mayLoad = 0; 611 let mayStore = 1; 612 let elements = getMUBUFElements<store_vt>.ret; 613 let tfe = isTFE; 614} 615 616multiclass MUBUF_Pseudo_Store_Pats_Common<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 617 618 def : GCNPat < 619 (st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 620 (!cast<MUBUF_Pseudo>(BaseInst # _OFFSET) store_vt:$vdata, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 621 622 def : GCNPat < 623 (st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 624 (!cast<MUBUF_Pseudo>(BaseInst # _ADDR64) store_vt:$vdata, i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)>; 625} 626 627multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SDPatternOperator st = null_frag> { 628 let SubtargetPredicate = HasUnrestrictedSOffset in { 629 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst, store_vt, st>; 630 } 631 defm : MUBUF_Pseudo_Store_Pats_Common<BaseInst # "_VBUFFER", store_vt, st>; 632} 633 634multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt, 635 bit isTFE, bit hasGFX12Enc> { 636 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 637 638 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasGFX12Enc>, 639 MUBUFAddr64Table<0, NAME>; 640 641 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasGFX12Enc>, 642 MUBUFAddr64Table<1, NAME>; 643 644 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasGFX12Enc>; 645 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasGFX12Enc>; 646 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasGFX12Enc>; 647 648 let DisableWQM = 1 in { 649 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasGFX12Enc>; 650 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasGFX12Enc>; 651 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasGFX12Enc>; 652 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasGFX12Enc>; 653 } 654} 655 656multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> { 657 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>; 658 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>; 659 660 defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>; 661 defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>; 662} 663 664class MUBUF_Pseudo_Store_Lds<string opName> 665 : MUBUF_Pseudo<opName, 666 (outs), 667 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, i1imm:$swz), 668 " $srsrc, $soffset$offset lds$cpol"> { 669 let LGKM_CNT = 1; 670 let mayLoad = 1; 671 let mayStore = 1; 672 673 let has_vdata = 0; 674 let has_vaddr = 0; 675 let lds = 1; 676 let VALU = 1; 677 678 let Uses = [EXEC, M0]; 679 let AsmMatchConverter = "cvtMubuf"; 680} 681 682class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, bit hasGFX12Enc, 683 list<RegisterClass> vaddrList=[]> { 684 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 685 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 686 687 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); 688 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); 689 dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset)); 690 dag MainInputs = !con((ins SReg_128:$srsrc), SOffset, (ins offset:$offset)); 691 dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol), 692 (ins CPol_NonGLC_WithDefault:$cpol)); 693 694 dag ret = !con(Data, MainInputs, CPol); 695} 696 697class getMUBUFAtomicIns<int addrKind, 698 RegisterClass vdataClass, 699 bit vdata_in, 700 bit hasGFX12Enc, 701 // Workaround bug bz30254 702 RegisterClass vdataClassCopy=vdataClass> { 703 dag ret = 704 !if(!eq(addrKind, BUFAddrKind.Offset), 705 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc>.ret, 706 !if(!eq(addrKind, BUFAddrKind.OffEn), 707 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VGPR_32]>.ret, 708 !if(!eq(addrKind, BUFAddrKind.IdxEn), 709 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VGPR_32]>.ret, 710 !if(!eq(addrKind, BUFAddrKind.BothEn), 711 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VReg_64]>.ret, 712 !if(!eq(addrKind, BUFAddrKind.Addr64), 713 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VReg_64]>.ret, 714 (ins)))))); 715} 716 717class MUBUF_Atomic_Pseudo<string opName, 718 int addrKind, 719 dag outs, 720 dag ins, 721 string asmOps, 722 list<dag> pattern=[], 723 // Workaround bug bz30254 724 int addrKindCopy = addrKind> 725 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 726 MUBUF_SetupAddr<addrKindCopy> { 727 let mayStore = 1; 728 let mayLoad = 1; 729 let hasPostISelHook = 1; 730 let hasSideEffects = 1; 731 let DisableWQM = 1; 732 let has_glc = 0; 733 let has_dlc = 0; 734 let has_sccb = 1; 735 let AsmMatchConverter = "cvtMubufAtomic"; 736} 737 738class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 739 RegisterClass vdataClass, 740 bit hasGFX12Enc = 0, 741 list<dag> pattern=[], 742 // Workaround bug bz30254 743 int addrKindCopy = addrKind, 744 RegisterClass vdataClassCopy = vdataClass> 745 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 746 (outs), 747 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0, hasGFX12Enc>.ret, 748 getMUBUFAsmOps<addrKindCopy>.ret, 749 pattern>, 750 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 751 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 752 let glc_value = 0; 753 let dlc_value = 0; 754 let sccb_value = 0; 755 let IsAtomicNoRet = 1; 756} 757 758class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 759 RegisterClass vdataClass, 760 bit hasGFX12Enc = 0, 761 list<dag> pattern=[], 762 // Workaround bug bz30254 763 int addrKindCopy = addrKind, 764 RegisterClass vdataClassCopy = vdataClass, 765 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 766 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 767 (outs vdata_op:$vdata), 768 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1, hasGFX12Enc>.ret, 769 getMUBUFAsmOps<addrKindCopy>.ret, 770 pattern>, 771 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 772 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 773 let glc_value = 1; 774 let dlc_value = 0; 775 let sccb_value = 0; 776 let IsAtomicRet = 1; 777 let Constraints = "$vdata = $vdata_in"; 778 let DisableEncoding = "$vdata_in"; 779} 780 781multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 782 RegisterClass vdataClass, 783 ValueType vdataType, 784 bit isFP = isFloatType<vdataType>.ret> { 785 let FPAtomic = isFP in { 786 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>, 787 MUBUFAddr64Table <0, NAME>; 788 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>, 789 MUBUFAddr64Table <1, NAME>; 790 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 791 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 792 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 793 794 def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>, 795 MUBUFAddr64Table <0, NAME # "_VBUFFER">; 796 def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>, 797 MUBUFAddr64Table <1, NAME # "_VBUFFER">; 798 def _VBUFFER_OFFEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 799 def _VBUFFER_IDXEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 800 def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 801 } 802} 803 804multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 805 RegisterClass vdataClass, 806 ValueType vdataType, 807 SDPatternOperator atomic, 808 bit isFP = isFloatType<vdataType>.ret> { 809 let FPAtomic = isFP in { 810 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0, 811 [(set vdataType:$vdata, 812 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 813 vdataType:$vdata_in))]>, 814 MUBUFAddr64Table <0, NAME # "_RTN">; 815 816 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0, 817 [(set vdataType:$vdata, 818 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 819 vdataType:$vdata_in))]>, 820 MUBUFAddr64Table <1, NAME # "_RTN">; 821 822 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, 0>; 823 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, 0>; 824 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>; 825 826 def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1, 827 [(set vdataType:$vdata, 828 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 829 vdataType:$vdata_in))]>, 830 MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">; 831 832 def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1, 833 [(set vdataType:$vdata, 834 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 835 vdataType:$vdata_in))]>, 836 MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">; 837 838 def _VBUFFER_OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn, vdataClass, 1>; 839 def _VBUFFER_IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn, vdataClass, 1>; 840 def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>; 841 } 842} 843 844multiclass MUBUF_Pseudo_Atomics <string opName, 845 RegisterClass vdataClass, 846 ValueType vdataType, 847 SDPatternOperator atomic = null_frag> : 848 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 849 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 850 851 852//===----------------------------------------------------------------------===// 853// MUBUF Instructions 854//===----------------------------------------------------------------------===// 855 856defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 857 "buffer_load_format_x", f32 858>; 859defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 860 "buffer_load_format_xy", v2f32 861>; 862defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 863 "buffer_load_format_xyz", v3f32 864>; 865defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 866 "buffer_load_format_xyzw", v4f32 867>; 868defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 869 "buffer_store_format_x", f32 870>; 871defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 872 "buffer_store_format_xy", v2f32 873>; 874defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 875 "buffer_store_format_xyz", v3f32 876>; 877defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 878 "buffer_store_format_xyzw", v4f32 879>; 880 881let OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1 in { 882let TiedSourceNotRead = 1 in { 883 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 884 "buffer_load_format_d16_x", i32 885 >; 886 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 887 "buffer_load_format_d16_xy", v2i32 888 >; 889 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 890 "buffer_load_format_d16_xyz", v3i32 891 >; 892 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 893 "buffer_load_format_d16_xyzw", v4i32 894 >; 895} 896 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 897 "buffer_store_format_d16_x", i32 898 >; 899 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 900 "buffer_store_format_d16_xy", v2i32 901 >; 902 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 903 "buffer_store_format_d16_xyz", v3i32 904 >; 905 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 906 "buffer_store_format_d16_xyzw", v4i32 907 >; 908} // End OtherPredicates = [HasUnpackedD16VMem], D16Buf = 1. 909 910let OtherPredicates = [HasPackedD16VMem], D16Buf = 1 in { 911let TiedSourceNotRead = 1 in { 912 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 913 "buffer_load_format_d16_x", f16 914 >; 915 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 916 "buffer_load_format_d16_xy", v2f16 917 >; 918 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 919 "buffer_load_format_d16_xyz", v3f16 920 >; 921 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 922 "buffer_load_format_d16_xyzw", v4f16 923 >; 924} 925 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 926 "buffer_store_format_d16_x", f16 927 >; 928 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 929 "buffer_store_format_d16_xy", v2f16 930 >; 931 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 932 "buffer_store_format_d16_xyz", v3f16 933 >; 934 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 935 "buffer_store_format_d16_xyzw", v4f16 936 >; 937} // End OtherPredicates = [HasPackedD16VMem], D16Buf = 1. 938 939defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 940 "buffer_load_ubyte", i32 941>; 942defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 943 "buffer_load_sbyte", i32 944>; 945defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 946 "buffer_load_ushort", i32 947>; 948defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 949 "buffer_load_sshort", i32 950>; 951defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 952 "buffer_load_dword", i32 953>; 954defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 955 "buffer_load_dwordx2", v2i32 956>; 957defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 958 "buffer_load_dwordx3", v3i32 959>; 960defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 961 "buffer_load_dwordx4", v4i32 962>; 963 964defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 965 "buffer_load_lds_b32", i32 966>; 967defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 968 "buffer_load_lds_format_x", f32 969>; 970defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 971 "buffer_load_lds_i8", i32 972>; 973defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 974 "buffer_load_lds_i16", i32 975>; 976defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 977 "buffer_load_lds_u8", i32 978>; 979defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 980 "buffer_load_lds_u16", i32 981>; 982 983defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 984defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 985defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 986defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 987defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 988defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 989defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 990defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 991defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 992defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 993 994foreach vt = Reg32Types.types in { 995defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", vt, load_global>; 996} 997 998foreach vt = VReg_64.RegTypes in { 999defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", vt, load_global>; 1000} 1001 1002foreach vt = VReg_96.RegTypes in { 1003defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", vt, load_global>; 1004} 1005 1006foreach vt = VReg_128.RegTypes in { 1007defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", vt, load_global>; 1008} 1009 1010defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 1011 "buffer_store_byte", i32 1012>; 1013defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 1014 "buffer_store_short", i32 1015>; 1016defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 1017 "buffer_store_dword", i32 1018>; 1019defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 1020 "buffer_store_dwordx2", v2i32 1021>; 1022defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 1023 "buffer_store_dwordx3", v3i32 1024>; 1025defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 1026 "buffer_store_dwordx4", v4i32 1027>; 1028 1029defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_BYTE", i32, truncstorei8_global>; 1030defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_SHORT", i32, truncstorei16_global>; 1031 1032foreach vt = Reg32Types.types in { 1033defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORD", vt, store_global>; 1034} 1035 1036foreach vt = VReg_64.RegTypes in { 1037defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX2", vt, store_global>; 1038} 1039 1040foreach vt = VReg_96.RegTypes in { 1041defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX3", vt, store_global>; 1042} 1043 1044foreach vt = VReg_128.RegTypes in { 1045defm : MUBUF_Pseudo_Store_Pats<"BUFFER_STORE_DWORDX4", vt, store_global>; 1046} 1047 1048defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 1049 "buffer_atomic_swap", VGPR_32, i32 1050>; 1051defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 1052 "buffer_atomic_cmpswap", VReg_64, v2i32 1053>; 1054defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 1055 "buffer_atomic_add", VGPR_32, i32 1056>; 1057defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 1058 "buffer_atomic_sub", VGPR_32, i32 1059>; 1060defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 1061 "buffer_atomic_smin", VGPR_32, i32 1062>; 1063defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 1064 "buffer_atomic_umin", VGPR_32, i32 1065>; 1066defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 1067 "buffer_atomic_smax", VGPR_32, i32 1068>; 1069defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 1070 "buffer_atomic_umax", VGPR_32, i32 1071>; 1072defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 1073 "buffer_atomic_and", VGPR_32, i32 1074>; 1075defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 1076 "buffer_atomic_or", VGPR_32, i32 1077>; 1078defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1079 "buffer_atomic_xor", VGPR_32, i32 1080>; 1081defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1082 "buffer_atomic_inc", VGPR_32, i32 1083>; 1084defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1085 "buffer_atomic_dec", VGPR_32, i32 1086>; 1087defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1088 "buffer_atomic_swap_x2", VReg_64, i64 1089>; 1090defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1091 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1092>; 1093defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1094 "buffer_atomic_add_x2", VReg_64, i64 1095>; 1096defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1097 "buffer_atomic_sub_x2", VReg_64, i64 1098>; 1099defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1100 "buffer_atomic_smin_x2", VReg_64, i64 1101>; 1102defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1103 "buffer_atomic_umin_x2", VReg_64, i64 1104>; 1105defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1106 "buffer_atomic_smax_x2", VReg_64, i64 1107>; 1108defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1109 "buffer_atomic_umax_x2", VReg_64, i64 1110>; 1111defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1112 "buffer_atomic_and_x2", VReg_64, i64 1113>; 1114defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1115 "buffer_atomic_or_x2", VReg_64, i64 1116>; 1117defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1118 "buffer_atomic_xor_x2", VReg_64, i64 1119>; 1120defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1121 "buffer_atomic_inc_x2", VReg_64, i64 1122>; 1123defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1124 "buffer_atomic_dec_x2", VReg_64, i64 1125>; 1126 1127let OtherPredicates = [HasGFX10_BEncoding] in { 1128 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics < 1129 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1130 >; 1131} 1132 1133let SubtargetPredicate = isGFX8GFX9 in { 1134def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1135} 1136 1137let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1138/* 1139defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1140defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1141*/ 1142 1143def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1144 int_amdgcn_buffer_wbinvl1_sc>; 1145} 1146 1147let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1148 1149defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1150 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1151>; 1152defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1153 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1154>; 1155defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1156 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1157>; 1158 1159} 1160 1161let SubtargetPredicate = isGFX6GFX7GFX10 in { 1162 1163defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1164 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1165>; 1166defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1167 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1168>; 1169defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1170 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1171>; 1172 1173} 1174 1175let SubtargetPredicate = HasD16LoadStore in { 1176let TiedSourceNotRead = 1 in { 1177 1178defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1179 "buffer_load_ubyte_d16", i32, 1 1180>; 1181 1182defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1183 "buffer_load_ubyte_d16_hi", i32, 1 1184>; 1185 1186defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1187 "buffer_load_sbyte_d16", i32, 1 1188>; 1189 1190defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1191 "buffer_load_sbyte_d16_hi", i32, 1 1192>; 1193 1194defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1195 "buffer_load_short_d16", i32, 1 1196>; 1197 1198defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1199 "buffer_load_short_d16_hi", i32, 1 1200>; 1201 1202defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1203 "buffer_load_format_d16_hi_x", i32 1204>; 1205} // End TiedSourceNotRead 1206 1207defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1208 "buffer_store_byte_d16_hi", i32 1209>; 1210 1211defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1212 "buffer_store_short_d16_hi", i32 1213>; 1214 1215defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1216 "buffer_store_format_d16_hi_x", i32 1217>; 1218 1219} // End HasD16LoadStore 1220 1221let SubtargetPredicate = isNotGFX12Plus in 1222def BUFFER_WBINVL1 : MUBUF_Invalidate < 1223 "buffer_wbinvl1", int_amdgcn_buffer_wbinvl1 1224>; 1225 1226let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1227defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1228 "buffer_atomic_add_f32", VGPR_32, f32 1229>; 1230 1231let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in 1232defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1233 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1234>; 1235 1236let OtherPredicates = [HasAtomicFaddRtnInsts] in 1237defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1238 "buffer_atomic_add_f32", VGPR_32, f32, null_frag 1239>; 1240 1241let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in 1242defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1243 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag 1244>; 1245 1246//===----------------------------------------------------------------------===// 1247// MTBUF Instructions 1248//===----------------------------------------------------------------------===// 1249 1250defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1251defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1252defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1253defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1254defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1255defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1256defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1257defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1258 1259let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1260let TiedSourceNotRead = 1 in { 1261 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1262 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1263 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1264 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1265} 1266 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1267 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1268 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1269 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1270} // End HasUnpackedD16VMem. 1271 1272let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1273let TiedSourceNotRead = 1 in { 1274 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1275 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1276 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1277 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1278} 1279 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1280 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1281 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1282 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1283} // End HasPackedD16VMem. 1284 1285let SubtargetPredicate = isGFX7Plus in { 1286 1287//===----------------------------------------------------------------------===// 1288// Instruction definitions for CI and newer. 1289//===----------------------------------------------------------------------===// 1290 1291def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1292 int_amdgcn_buffer_wbinvl1_vol>; 1293 1294} // End let SubtargetPredicate = isGFX7Plus 1295 1296let SubtargetPredicate = isGFX90APlus in { 1297 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1298 let has_glc = 1; 1299 let has_sccb = 1; 1300 let InOperandList = (ins CPol_0:$cpol); 1301 let AsmOperands = "$cpol"; 1302 } 1303 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1304 let SubtargetPredicate = isGFX90AOnly; 1305 } 1306 1307 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1308 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1309 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1310} // End SubtargetPredicate = isGFX90APlus 1311 1312def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1313 let SubtargetPredicate = isGFX940Plus; 1314 let has_glc = 1; 1315 let has_sccb = 1; 1316 let InOperandList = (ins CPol_0:$cpol); 1317 let AsmOperands = "$cpol"; 1318} 1319 1320def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1321def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1322 1323//===----------------------------------------------------------------------===// 1324// MUBUF Patterns 1325//===----------------------------------------------------------------------===// 1326 1327//===----------------------------------------------------------------------===// 1328// buffer_load/store_format patterns 1329//===----------------------------------------------------------------------===// 1330 1331multiclass MUBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1332 string opcode, ValueType memoryVt = vt> { 1333 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1334 1335 def : GCNPat< 1336 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1337 timm:$auxiliary, 0)), 1338 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1339 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1340 >; 1341 1342 def : GCNPat< 1343 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1344 timm:$auxiliary, 0)), 1345 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1346 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1347 >; 1348 1349 def : GCNPat< 1350 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1351 timm:$auxiliary, timm)), 1352 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1353 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1354 >; 1355 1356 def : GCNPat< 1357 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1358 timm:$auxiliary, timm)), 1359 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1360 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1361 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1362 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1363 >; 1364} 1365 1366multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1367 string opcode, ValueType memoryVt = vt>{ 1368 let SubtargetPredicate = HasUnrestrictedSOffset in { 1369 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1370 } 1371 defm : MUBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1372} 1373 1374defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1375defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1376defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1377defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1378defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1379defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1380defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1381defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1382 1383defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">; 1384defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">; 1385defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">; 1386defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">; 1387 1388let OtherPredicates = [HasUnpackedD16VMem] in { 1389 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1390 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1391 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1392 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1393 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1394 defm : MUBUF_LoadIntrinsicPat_Common<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1395} // End HasUnpackedD16VMem. 1396 1397let OtherPredicates = [HasPackedD16VMem] in { 1398 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1399 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1400 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1401 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1402 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1403 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1404 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1405 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1406 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1407} // End HasPackedD16VMem. 1408 1409defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1410defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1411defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1412defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1413defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1414defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1415defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1416defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1417defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1418defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1419defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1420defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1421defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1422defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1423defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1424defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1425 1426multiclass MUBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 1427 string opcode, ValueType memoryVt = vt> { 1428 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1429 1430 def : GCNPat< 1431 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 1432 timm:$auxiliary, 0), 1433 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1434 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1435 >; 1436 1437 def : GCNPat< 1438 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1439 timm:$auxiliary, 0), 1440 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1441 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1442 >; 1443 1444 def : GCNPat< 1445 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 1446 timm:$auxiliary, timm), 1447 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1448 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1449 >; 1450 1451 def : GCNPat< 1452 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1453 timm:$auxiliary, timm), 1454 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1455 getVregSrcForVT<vt>.ret:$vdata, 1456 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1457 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary), 1458 (extract_swz $auxiliary)) 1459 >; 1460} 1461 1462multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1463 string opcode, ValueType memoryVt = vt> { 1464 let SubtargetPredicate = HasUnrestrictedSOffset in { 1465 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 1466 } 1467 defm : MUBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 1468} 1469 1470defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1471defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1472defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1473defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1474defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1475defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1476defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1477defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1478 1479let OtherPredicates = [HasUnpackedD16VMem] in { 1480 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1481 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1482 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1483 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1484 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1485 defm : MUBUF_StoreIntrinsicPat_Common<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1486} // End HasUnpackedD16VMem. 1487 1488let OtherPredicates = [HasPackedD16VMem] in { 1489 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1490 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1491 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1492 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1493 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1494 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1495 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1496 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1497 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1498} // End HasPackedD16VMem. 1499 1500defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1501defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1502defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1503defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1504defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1505defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1506defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1507defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1508defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1509defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1510defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1511defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1512defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1513defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1514 1515//===----------------------------------------------------------------------===// 1516// buffer_atomic patterns 1517//===----------------------------------------------------------------------===// 1518 1519multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1520 foreach RtnMode = ["ret", "noret"] in { 1521 1522 defvar Op = !cast<SDPatternOperator>(OpPrefix 1523 # !if(!eq(RtnMode, "ret"), "", "_noret") 1524 # !if(isIntr, "", "_" # vt.Size)); 1525 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1526 1527 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1528 def : GCNPat< 1529 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)), 1530 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1531 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1532 >; 1533 1534 def : GCNPat< 1535 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1536 vt:$vdata_in)), 1537 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1538 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1539 >; 1540 } // end let AddedComplexity 1541 1542 } // end foreach RtnMode 1543} 1544 1545multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1546 let SubtargetPredicate = HasUnrestrictedSOffset in { 1547 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst, isIntr>; 1548 } 1549 defm : BufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", isIntr>; 1550} 1551 1552multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1553 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1554} 1555 1556multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1557 foreach RtnMode = ["ret", "noret"] in { 1558 1559 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1560 # !if(!eq(RtnMode, "ret"), "", "_noret") 1561 # "_" # vt.Size); 1562 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1563 1564 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1565 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1566 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1567 offset:$offset); 1568 def : GCNPat< 1569 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)), 1570 !if(!eq(RtnMode, "ret"), 1571 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1572 !if(!eq(vt, i32), sub0, sub0_sub1)), 1573 OffsetResDag) 1574 >; 1575 1576 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1577 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1578 SCSrc_b32:$soffset, offset:$offset); 1579 def : GCNPat< 1580 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1581 data_vt:$vdata_in)), 1582 !if(!eq(RtnMode, "ret"), 1583 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1584 !if(!eq(vt, i32), sub0, sub0_sub1)), 1585 Addr64ResDag) 1586 >; 1587 } // end let AddedComplexity 1588 1589 } // end foreach RtnMode 1590} 1591 1592multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1593 let SubtargetPredicate = HasUnrestrictedSOffset in { 1594 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1595 } 1596 defm : BufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1597} 1598 1599 1600foreach Ty = [i32, i64] in { 1601 1602defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1603 1604defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1605defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1606defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1607defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1608defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1609defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1610defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1611defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1612defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1613defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1614defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1615defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1616 1617} // end foreach Ty 1618 1619defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1620defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1621 1622multiclass SIBufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, 1623 list<string> RtnModes = ["ret", "noret"]> { 1624 foreach RtnMode = RtnModes in { 1625 1626 defvar Op = !cast<SDPatternOperator>(OpPrefix 1627 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1628 1629 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1630 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1631 (set_glc $cachepolicy), (timm:$cachepolicy)); 1632 1633 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1634 def : GCNPat< 1635 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1636 timm:$offset, timm:$cachepolicy, 0)), 1637 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1638 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1639 timm:$offset, CachePolicy) 1640 >; 1641 1642 def : GCNPat< 1643 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), 1644 timm:$offset, timm:$cachepolicy, timm)), 1645 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1646 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1647 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1648 >; 1649 1650 def : GCNPat< 1651 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1652 (BUFSOffset i32:$soffset), timm:$offset, timm:$cachepolicy, 0)), 1653 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1654 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1655 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1656 >; 1657 1658 def : GCNPat< 1659 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1660 (BUFSOffset i32:$soffset), timm:$offset, timm:$cachepolicy, timm)), 1661 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1662 getVregSrcForVT<vt>.ret:$vdata_in, 1663 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1664 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1665 >; 1666 } // end let AddedComplexity 1667 1668 } // end foreach RtnMode 1669} 1670 1671multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1672 list<string> RtnModes = ["ret", "noret"]> { 1673 let SubtargetPredicate = HasUnrestrictedSOffset in { 1674 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst, RtnModes>; 1675 } 1676 defm : SIBufferAtomicPat_Common<OpPrefix, vt, Inst # "_VBUFFER", RtnModes>; 1677} 1678 1679defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1680defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1681defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1682defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1683defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1684defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1685defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1686defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1687defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1688defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1689defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1690defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1691defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1692defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1693defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1694defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1695defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1696defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1697defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1698defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1699defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1700defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1701defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1702defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1703defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1704defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1705 1706let SubtargetPredicate = HasAtomicCSubNoRtnInsts in 1707defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["noret"]>; 1708 1709let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1710 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1711 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1712} 1713let SubtargetPredicate = isGFX6GFX7GFX10 in { 1714 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1715 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1716} 1717 1718class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1719 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1720 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> { 1721 let HasNoUse = true; 1722} 1723 1724multiclass BufferAtomicPatterns_NO_RTN_Common<SDPatternOperator name, ValueType vt, 1725 string opcode> { 1726 def : GCNPat< 1727 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1728 0, (BUFSOffset i32:$soffset), timm:$offset, 1729 timm:$cachepolicy, 0), 1730 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1731 timm:$offset, timm:$cachepolicy) 1732 >; 1733 1734 def : GCNPat< 1735 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1736 0, (BUFSOffset i32:$soffset), timm:$offset, 1737 timm:$cachepolicy, timm), 1738 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1739 timm:$offset, timm:$cachepolicy) 1740 >; 1741 1742 def : GCNPat< 1743 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1744 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1745 timm:$cachepolicy, 0), 1746 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1747 timm:$offset, timm:$cachepolicy) 1748 >; 1749 1750 def : GCNPat< 1751 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1752 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1753 timm:$cachepolicy, timm), 1754 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1755 getVregSrcForVT<vt>.ret:$vdata_in, 1756 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1757 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, timm:$cachepolicy) 1758 >; 1759} 1760 1761multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1762 string opcode> { 1763 let SubtargetPredicate = HasUnrestrictedSOffset in { 1764 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode>; 1765 } 1766 defm : BufferAtomicPatterns_NO_RTN_Common<name, vt, opcode # "_VBUFFER">; 1767} 1768 1769let OtherPredicates = [HasAtomicFaddNoRtnInsts] in 1770 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>; 1771 1772let OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] in { 1773 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>; 1774} // End OtherPredicates = [HasAtomicBufferGlobalPkAddF16NoRtnInsts] 1775 1776let OtherPredicates = [HasAtomicFaddRtnInsts] in 1777 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>; 1778 1779let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in { 1780 defm : SIBufferAtomicPat_Common<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>; 1781} // End OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] 1782 1783let OtherPredicates = [isGFX90APlus] in { 1784 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1785 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1786 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1787} // End SubtargetPredicate = isGFX90APlus 1788 1789multiclass SIBufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string Inst> { 1790 foreach RtnMode = ["ret", "noret"] in { 1791 defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1792 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1793 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1794 defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1795 (timm:$cachepolicy)); 1796 defvar SrcRC = getVregSrcForVT<vt>.ret; 1797 defvar DataRC = getVregSrcForVT<data_vt>.ret; 1798 defvar SubLo = !if(!eq(vt, i32), sub0, sub0_sub1); 1799 defvar SubHi = !if(!eq(vt, i32), sub1, sub2_sub3); 1800 1801 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1802 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1803 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1804 def : GCNPat< 1805 (vt (Op 1806 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), 1807 timm:$offset, timm:$cachepolicy, 0)), 1808 !if(!eq(RtnMode, "ret"), 1809 (EXTRACT_SUBREG OffsetResDag, SubLo), 1810 OffsetResDag) 1811 >; 1812 1813 defvar IdxenResDag = (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1814 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1815 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1816 CachePolicy); 1817 def : GCNPat< 1818 (vt (Op 1819 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1820 0, (BUFSOffset i32:$soffset), timm:$offset, 1821 timm:$cachepolicy, timm)), 1822 !if(!eq(RtnMode, "ret"), 1823 (EXTRACT_SUBREG IdxenResDag, SubLo), 1824 IdxenResDag) 1825 >; 1826 1827 defvar OffenResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1828 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1829 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1830 CachePolicy); 1831 def : GCNPat< 1832 (vt (Op 1833 vt:$data, vt:$cmp, v4i32:$rsrc, 0, 1834 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1835 timm:$cachepolicy, 0)), 1836 !if(!eq(RtnMode, "ret"), 1837 (EXTRACT_SUBREG OffenResDag, SubLo), 1838 OffenResDag) 1839 >; 1840 1841 defvar BothenResDag = (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1842 (REG_SEQUENCE DataRC, SrcRC:$data, SubLo, SrcRC:$cmp, SubHi), 1843 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1844 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1845 def : GCNPat< 1846 (vt (Op 1847 vt:$data, vt:$cmp, v4i32:$rsrc, i32:$vindex, 1848 i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 1849 timm:$cachepolicy, timm)), 1850 !if(!eq(RtnMode, "ret"), 1851 (EXTRACT_SUBREG BothenResDag, SubLo), 1852 BothenResDag) 1853 >; 1854 } // end foreach RtnMode 1855} 1856 1857multiclass SIBufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1858 let SubtargetPredicate = HasUnrestrictedSOffset in { 1859 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst>; 1860 } 1861 defm : SIBufferAtomicCmpSwapPat_Common<vt, data_vt, Inst # "_VBUFFER">; 1862} 1863 1864defm : SIBufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1865defm : SIBufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1866 1867class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1868 PatFrag constant_ld> : GCNPat < 1869 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1870 i32:$offset))), 1871 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1872 >; 1873 1874multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1875 ValueType vt, PatFrag atomic_ld> { 1876 def : GCNPat < 1877 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 1878 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1879 >; 1880 1881 def : GCNPat < 1882 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), 1883 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1884 >; 1885} 1886 1887let SubtargetPredicate = isGFX6GFX7 in { 1888def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1889def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1890def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1891def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1892def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1893def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1894 1895defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1896defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1897} // End SubtargetPredicate = isGFX6GFX7 1898 1899multiclass MUBUFLoad_PatternOffset_Common <string Instr, ValueType vt, 1900 PatFrag ld> { 1901 def : GCNPat < 1902 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1903 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $srsrc, $soffset, $offset) 1904 >; 1905} 1906 1907multiclass MUBUFLoad_PatternOffset <string Instr, ValueType vt, 1908 PatFrag ld> { 1909 let SubtargetPredicate = HasUnrestrictedSOffset in { 1910 defm : MUBUFLoad_PatternOffset_Common<Instr, vt, ld>; 1911 } 1912 defm : MUBUFLoad_PatternOffset_Common<Instr # "_VBUFFER", vt, ld>; 1913} 1914 1915let OtherPredicates = [Has16BitInsts] in { 1916 1917defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_constant>; 1918defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_constant>; 1919defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_constant>; 1920defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_SBYTE", i16, sextloadi8_global>; 1921defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, extloadi8_global>; 1922defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_UBYTE", i16, zextloadi8_global>; 1923 1924defm : MUBUFLoad_PatternOffset <"BUFFER_LOAD_USHORT", i16, load_global>; 1925 1926} // End OtherPredicates = [Has16BitInsts] 1927 1928multiclass MUBUFScratchLoadPat_Common <string Instr, 1929 ValueType vt, PatFrag ld> { 1930 def : GCNPat < 1931 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1932 i32:$soffset, i32:$offset))), 1933 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, 0, 0) 1934 >; 1935 1936 def : GCNPat < 1937 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1938 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, 0, 0) 1939 >; 1940} 1941 1942multiclass MUBUFScratchLoadPat <string Instr, 1943 ValueType vt, PatFrag ld> { 1944 let SubtargetPredicate = HasUnrestrictedSOffset in { 1945 defm : MUBUFScratchLoadPat_Common<Instr, vt, ld>; 1946 } 1947 defm : MUBUFScratchLoadPat_Common<Instr # "_VBUFFER", vt, ld>; 1948} 1949 1950// XXX - Is it possible to have a complex pattern in a PatFrag? 1951multiclass MUBUFScratchLoadPat_D16_Common <string Instr, 1952 ValueType vt, PatFrag ld_frag> { 1953 def : GCNPat < 1954 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in), 1955 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) $vaddr, $srsrc, $soffset, $offset, $in) 1956 >; 1957 1958 def : GCNPat < 1959 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in), 1960 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) $srsrc, $soffset, $offset, $in) 1961 >; 1962} 1963 1964multiclass MUBUFScratchLoadPat_D16 <string Instr, 1965 ValueType vt, PatFrag ld_frag> { 1966 let SubtargetPredicate = HasUnrestrictedSOffset in { 1967 defm : MUBUFScratchLoadPat_D16_Common<Instr, vt, ld_frag>; 1968 } 1969 defm : MUBUFScratchLoadPat_D16_Common<Instr # "_VBUFFER", vt, ld_frag>; 1970} 1971 1972let OtherPredicates = [DisableFlatScratch] in { 1973defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i32, sextloadi8_private>; 1974defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, extloadi8_private>; 1975defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i32, zextloadi8_private>; 1976defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SBYTE", i16, sextloadi8_private>; 1977defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, extloadi8_private>; 1978defm : MUBUFScratchLoadPat <"BUFFER_LOAD_UBYTE", i16, zextloadi8_private>; 1979defm : MUBUFScratchLoadPat <"BUFFER_LOAD_SSHORT", i32, sextloadi16_private>; 1980defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, extloadi16_private>; 1981defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i32, zextloadi16_private>; 1982defm : MUBUFScratchLoadPat <"BUFFER_LOAD_USHORT", i16, load_private>; 1983 1984foreach vt = Reg32Types.types in { 1985defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORD", vt, load_private>; 1986} 1987defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX2", v2i32, load_private>; 1988defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX3", v3i32, load_private>; 1989defm : MUBUFScratchLoadPat <"BUFFER_LOAD_DWORDX4", v4i32, load_private>; 1990 1991let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1992defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2i16, load_d16_hi_private>; 1993defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2i16, az_extloadi8_d16_hi_private>; 1994defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2i16, sextloadi8_d16_hi_private>; 1995defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16_HI", v2f16, load_d16_hi_private>; 1996defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16_HI", v2f16, az_extloadi8_d16_hi_private>; 1997defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16_HI", v2f16, sextloadi8_d16_hi_private>; 1998 1999defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2i16, load_d16_lo_private>; 2000defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2i16, az_extloadi8_d16_lo_private>; 2001defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2i16, sextloadi8_d16_lo_private>; 2002defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SHORT_D16", v2f16, load_d16_lo_private>; 2003defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_UBYTE_D16", v2f16, az_extloadi8_d16_lo_private>; 2004defm : MUBUFScratchLoadPat_D16<"BUFFER_LOAD_SBYTE_D16", v2f16, sextloadi8_d16_lo_private>; 2005} 2006 2007} // End OtherPredicates = [DisableFlatScratch] 2008 2009multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 2010 ValueType vt, PatFrag atomic_st> { 2011 def : GCNPat < 2012 (atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)), 2013 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 2014 >; 2015 2016 def : GCNPat < 2017 (atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)), 2018 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 2019 >; 2020} 2021let SubtargetPredicate = isGFX6GFX7 in { 2022defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 2023defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 2024defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 2025defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 2026defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 2027defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 2028} // End Predicates = isGFX6GFX7 2029 2030 2031multiclass MUBUFStore_PatternOffset_Common <string Instr, ValueType vt, 2032 PatFrag st> { 2033 2034 def : GCNPat < 2035 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 2036 (!cast<MUBUF_Pseudo>(Instr # "_OFFSET") $vdata, $srsrc, $soffset, $offset) 2037 >; 2038} 2039 2040multiclass MUBUFStore_PatternOffset <string Instr, ValueType vt, 2041 PatFrag st> { 2042 let SubtargetPredicate = HasUnrestrictedSOffset in { 2043 defm : MUBUFStore_PatternOffset_Common<Instr, vt, st>; 2044 } 2045 defm : MUBUFStore_PatternOffset_Common<Instr # "_VBUFFER", vt, st>; 2046} 2047 2048defm : MUBUFStore_PatternOffset <"BUFFER_STORE_BYTE", i16, truncstorei8_global>; 2049defm : MUBUFStore_PatternOffset <"BUFFER_STORE_SHORT", i16, store_global>; 2050 2051multiclass MUBUFScratchStorePat_Common <string Instr, 2052 ValueType vt, PatFrag st, 2053 RegisterClass rc = VGPR_32> { 2054 def : GCNPat < 2055 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 2056 i32:$soffset, i32:$offset)), 2057 (!cast<MUBUF_Pseudo>(Instr # _OFFEN) rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) 2058 >; 2059 2060 def : GCNPat < 2061 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 2062 i32:$offset)), 2063 (!cast<MUBUF_Pseudo>(Instr # _OFFSET) rc:$value, $srsrc, $soffset, $offset, 0, 0) 2064 >; 2065} 2066 2067multiclass MUBUFScratchStorePat <string Instr, 2068 ValueType vt, PatFrag st, 2069 RegisterClass rc = VGPR_32> { 2070 let SubtargetPredicate = HasUnrestrictedSOffset in { 2071 defm : MUBUFScratchStorePat_Common<Instr, vt, st, rc>; 2072 } 2073 defm : MUBUFScratchStorePat_Common<Instr # "_VBUFFER", vt, st, rc>; 2074} 2075 2076let OtherPredicates = [DisableFlatScratch] in { 2077defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i32, truncstorei8_private>; 2078defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i32, truncstorei16_private>; 2079defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE", i16, truncstorei8_private>; 2080defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT", i16, store_private>; 2081 2082foreach vt = Reg32Types.types in { 2083defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORD", vt, store_private>; 2084} 2085 2086defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX2", v2i32, store_private, VReg_64>; 2087defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX3", v3i32, store_private, VReg_96>; 2088defm : MUBUFScratchStorePat <"BUFFER_STORE_DWORDX4", v4i32, store_private, VReg_128>; 2089 2090 2091let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 2092 // Hiding the extract high pattern in the PatFrag seems to not 2093 // automatically increase the complexity. 2094let AddedComplexity = 1 in { 2095defm : MUBUFScratchStorePat <"BUFFER_STORE_SHORT_D16_HI", i32, store_hi16_private>; 2096defm : MUBUFScratchStorePat <"BUFFER_STORE_BYTE_D16_HI", i32, truncstorei8_hi16_private>; 2097} 2098} 2099} // End OtherPredicates = [DisableFlatScratch] 2100 2101//===----------------------------------------------------------------------===// 2102// MTBUF Patterns 2103//===----------------------------------------------------------------------===// 2104 2105//===----------------------------------------------------------------------===// 2106// tbuffer_load/store_format patterns 2107//===----------------------------------------------------------------------===// 2108 2109multiclass MTBUF_LoadIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2110 string opcode, ValueType memoryVt = vt> { 2111 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 2112 2113 def : GCNPat< 2114 (vt (st v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2115 timm:$format, timm:$auxiliary, 0)), 2116 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2117 (as_i8timm $format), 2118 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2119 >; 2120 2121 def : GCNPat< 2122 (vt (st v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2123 timm:$format, timm:$auxiliary, timm)), 2124 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2125 (as_i8timm $format), 2126 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2127 >; 2128 2129 def : GCNPat< 2130 (vt (st v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2131 timm:$format, timm:$auxiliary, 0)), 2132 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2133 (as_i8timm $format), 2134 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2135 >; 2136 2137 def : GCNPat< 2138 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2139 timm:$format, timm:$auxiliary, timm)), 2140 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 2141 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2142 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 2143 (as_i8timm $format), 2144 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2145 >; 2146} 2147 2148multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 2149 string opcode, ValueType memoryVt = vt> { 2150 let SubtargetPredicate = HasUnrestrictedSOffset in { 2151 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2152 } 2153 defm : MTBUF_LoadIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2154} 2155 2156defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 2157defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 2158defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 2159defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 2160defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 2161defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 2162defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 2163defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 2164 2165let OtherPredicates = [HasUnpackedD16VMem] in { 2166 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2167 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 2168 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 2169 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 2170 defm : MTBUF_LoadIntrinsicPat_Common<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 2171} // End HasUnpackedD16VMem. 2172 2173let OtherPredicates = [HasPackedD16VMem] in { 2174 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 2175 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 2176 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 2177 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 2178 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 2179} // End HasPackedD16VMem. 2180 2181multiclass MTBUF_StoreIntrinsicPat_Common<SDPatternOperator name, ValueType vt, 2182 string opcode, ValueType memoryVt = vt> { 2183 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 2184 2185 def : GCNPat< 2186 (st vt:$vdata, v4i32:$rsrc, 0, 0, (BUFSOffset i32:$soffset), timm:$offset, 2187 timm:$format, timm:$auxiliary, 0), 2188 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 2189 timm:$offset, (as_i8timm $format), 2190 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2191 >; 2192 2193 def : GCNPat< 2194 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, (BUFSOffset i32:$soffset), timm:$offset, 2195 timm:$format, timm:$auxiliary, timm), 2196 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 2197 timm:$offset, (as_i8timm $format), 2198 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2199 >; 2200 2201 def : GCNPat< 2202 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, (BUFSOffset i32:$soffset), timm:$offset, 2203 timm:$format, timm:$auxiliary, 0), 2204 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2205 timm:$offset, (as_i8timm $format), 2206 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2207 >; 2208 2209 def : GCNPat< 2210 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, (BUFSOffset i32:$soffset), 2211 timm:$offset, timm:$format, timm:$auxiliary, timm), 2212 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2213 getVregSrcForVT<vt>.ret:$vdata, 2214 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2215 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), 2216 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2217 >; 2218} 2219 2220multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 2221 string opcode, ValueType memoryVt = vt> { 2222 let SubtargetPredicate = HasUnrestrictedSOffset in { 2223 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode, memoryVt>; 2224 } 2225 defm : MTBUF_StoreIntrinsicPat_Common<name, vt, opcode # "_VBUFFER", memoryVt>; 2226} 2227 2228defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2229defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2230defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2231defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2232defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2233defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2234defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2235defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2236 2237let OtherPredicates = [HasUnpackedD16VMem] in { 2238 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2239 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2240 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2241 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2242 defm : MTBUF_StoreIntrinsicPat_Common<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2243} // End HasUnpackedD16VMem. 2244 2245let OtherPredicates = [HasPackedD16VMem] in { 2246 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2247 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2248 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2249 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2250 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2251} // End HasPackedD16VMem. 2252 2253//===----------------------------------------------------------------------===// 2254// Target-specific instruction encodings. 2255//===----------------------------------------------------------------------===// 2256 2257//===----------------------------------------------------------------------===// 2258// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2259//===----------------------------------------------------------------------===// 2260 2261class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2262 string real_name = ps.Mnemonic> : 2263 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2264 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2265 let Inst{31-26} = 0x38; 2266 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2267 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2268 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2269 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2270} 2271 2272class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2273 string real_name = ps.Mnemonic> : 2274 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2275 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2276 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2277 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2278 let Inst{25-18} = op; 2279 let Inst{53} = ps.tfe; 2280 let Inst{54} = ps.offen; 2281 let Inst{55} = ps.idxen; 2282 let SubtargetPredicate = isGFX11Only; 2283} 2284 2285class Base_MUBUF_Real_Atomic_gfx11<bits<8> op, MUBUF_Pseudo ps, 2286 string real_name> : 2287 MUBUF_Real_gfx11<op, ps, real_name> { 2288 let Inst{13} = cpol{CPolBit.DLC}; 2289} 2290 2291class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2292 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2293 let Inst{12} = ps.offen; 2294 let Inst{13} = ps.idxen; 2295 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2296 let Inst{16} = ps.lds; 2297 let Inst{24-18} = op; 2298 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2299 let Inst{55} = ps.tfe; 2300} 2301 2302class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2303 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2304 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2305 let Inst{25} = op{7}; 2306 let SubtargetPredicate = isGFX10Only; 2307} 2308 2309class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2310 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2311 let Inst{15} = ps.addr64; 2312 let SubtargetPredicate = isGFX6GFX7; 2313} 2314 2315//===----------------------------------------------------------------------===// 2316// Base ENC_VBUFFER for GFX12. 2317//===----------------------------------------------------------------------===// 2318 2319class VBUFFER_Real <BUF_Pseudo ps, string real_name = ps.Mnemonic> : 2320 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 { 2321 2322 let isPseudo = 0; 2323 let isCodeGenOnly = 0; 2324 2325 let VM_CNT = 1; 2326 let EXP_CNT = 1; 2327 2328 // copy relevant pseudo op flags 2329 let SubtargetPredicate = ps.SubtargetPredicate; 2330 let AsmMatchConverter = ps.AsmMatchConverter; 2331 let OtherPredicates = ps.OtherPredicates; 2332 let Constraints = ps.Constraints; 2333 let DisableEncoding = ps.DisableEncoding; 2334 let TSFlags = ps.TSFlags; 2335 let UseNamedOperandTable = ps.UseNamedOperandTable; 2336 let SchedRW = ps.SchedRW; 2337 let mayLoad = ps.mayLoad; 2338 let mayStore = ps.mayStore; 2339 let IsAtomicRet = ps.IsAtomicRet; 2340 let IsAtomicNoRet = ps.IsAtomicNoRet; 2341 let VALU = ps.VALU; 2342 let LGKM_CNT = ps.LGKM_CNT; 2343 2344 bits<24> offset; 2345 bits<8> vaddr; 2346 bits<10> vdata; 2347 2348 bits<7> srsrc; 2349 bits<7> soffset; 2350 bits<6> cpol; 2351 2352 let Inst{95-72} = !if(ps.has_offset, offset, ?); 2353 let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?); 2354 let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?); 2355 2356 let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?); 2357 let Inst{49-48} = 0b00; 2358 let Inst{6-0} = !if(ps.has_soffset, soffset, ?); 2359 let Inst{22} = ps.tfe; 2360 let Inst{62} = ps.offen; 2361 let Inst{63} = ps.idxen; 2362 2363 let Inst{54-53} = cpol{2-1}; // th{2-1} 2364 let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0} 2365 let Inst{51-50} = cpol{4-3}; // scope 2366 2367 let Inst{31-26} = 0b110001; 2368} 2369 2370class VBUFFER_MUBUF_Real_gfx12<bits<8> op, MUBUF_Pseudo ps, 2371 string real_name = ps.Mnemonic> : 2372 VBUFFER_Real<ps, real_name>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> { 2373 2374 let MUBUF = 1; 2375 2376 // Set the last bit of format to 1 to avoid round-trip issues, as some tools 2377 // print BUF_FMT_INVALID for format 0. 2378 let Inst{55} = 0b1; 2379 let Inst{21-14} = op; 2380 let SubtargetPredicate = isGFX12Only; 2381} 2382 2383class VBUFFER_MTBUF_Real_gfx12<bits<4> op, MTBUF_Pseudo ps, 2384 string real_name = ps.Mnemonic> : 2385 VBUFFER_Real<ps, real_name>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> { 2386 2387 let MTBUF = 1; 2388 2389 bits<7> format; 2390 2391 let Inst{17-14} = op; 2392 let Inst{21-18} = 0b1000; 2393 let Inst{61-55} = format; 2394} 2395 2396//===----------------------------------------------------------------------===// 2397// MUBUF - GFX11, GFX12. 2398//===----------------------------------------------------------------------===// 2399 2400// Shortcut to default Mnemonic from MUBUF_Pseudo. Hides the cast to the 2401// specific pseudo (bothen in this case) since any of them will work. 2402class get_MUBUF_ps<string name> { 2403 string Mnemonic = !cast<MUBUF_Pseudo>(name # "_BOTHEN").Mnemonic; 2404} 2405 2406// gfx11 instruction that accept both old and new assembler name. 2407class Mnem_gfx11_gfx12 <string mnemonic, string real_name> : 2408 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>; 2409 2410class Mnem_gfx11 <string mnemonic, string real_name> : 2411 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Only]>; 2412 2413class Mnem_gfx12 <string mnemonic, string real_name> : 2414 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX12Plus]>; 2415 2416class MUBUF_Real_gfx11_impl<bits<8> op, string ps_name, string real_name> : 2417 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2418 2419class VBUFFER_MUBUF_Real_gfx12_impl<bits<8> op, string ps_name, string real_name> : 2420 VBUFFER_MUBUF_Real_gfx12<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2421 2422multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<bits<8> op, string real_name> { 2423 let DecoderNamespace = "GFX11" in { 2424 def _BOTHEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_BOTHEN", real_name>; 2425 def _IDXEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_IDXEN", real_name>; 2426 def _OFFEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFEN", real_name>; 2427 def _OFFSET_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFSET", real_name>; 2428 } 2429} 2430 2431multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2<bits<8> op, string real_name> { 2432 let DecoderNamespace = "GFX12" in { 2433 def _BOTHEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN", real_name>; 2434 def _IDXEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_IDXEN", real_name>; 2435 def _OFFEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_OFFEN", real_name>; 2436 def _OFFSET_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_OFFSET", real_name>; 2437 } 2438} 2439 2440multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<bits<8> op, string real_name> : 2441 MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>, 2442 MUBUF_Real_AllAddr_gfx12_Renamed_Impl2<op, real_name>; 2443 2444multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name, 2445 bit hasTFE = 1> { 2446 defm NAME : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2447 if hasTFE then 2448 defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2449} 2450 2451multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<bits<8> op, string real_name, 2452 bit hasTFE = 1> { 2453 defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<op, real_name>; 2454 if hasTFE then 2455 defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<op, real_name>; 2456} 2457 2458// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions. 2459multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> : 2460 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>; 2461 2462multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op, bit hasTFE = 1> : 2463 MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>; 2464 2465multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<bits<8> op, string real_name> : 2466 MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, real_name> { 2467 def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2468} 2469 2470class MUBUF_Real_Atomic_gfx11_impl<bits<8> op, string ps_name, 2471 string real_name> : 2472 Base_MUBUF_Real_Atomic_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2473 2474class MUBUF_Real_Atomic_gfx12_impl<bits<8> op, string ps_name, 2475 string real_name> : 2476 VBUFFER_MUBUF_Real_gfx12<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2477 2478multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return, 2479 string real_name> { 2480 let DecoderNamespace = "GFX11" in { 2481 defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); 2482 def _BOTHEN#Rtn#_gfx11 : 2483 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_BOTHEN" # Rtn, real_name>, 2484 AtomicNoRet<NAME # "_BOTHEN_gfx11", is_return>; 2485 def _IDXEN#Rtn#_gfx11 : 2486 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_IDXEN" # Rtn, real_name>, 2487 AtomicNoRet<NAME # "_IDXEN_gfx11", is_return>; 2488 def _OFFEN#Rtn#_gfx11 : 2489 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFEN" # Rtn, real_name>, 2490 AtomicNoRet<NAME # "_OFFEN_gfx11", is_return>; 2491 def _OFFSET#Rtn#_gfx11 : 2492 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFSET" # Rtn, real_name>, 2493 AtomicNoRet<NAME # "_OFFSET_gfx11", is_return>; 2494 } 2495} 2496 2497multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl<bits<8> op, bit is_return, 2498 string real_name> { 2499 let DecoderNamespace = "GFX12" in { 2500 defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); 2501 def _BOTHEN#Rtn#_gfx12 : 2502 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN" # Rtn, real_name>, 2503 AtomicNoRet<NAME # "_BOTHEN_gfx12", is_return>; 2504 def _IDXEN#Rtn#_gfx12 : 2505 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_IDXEN" # Rtn, real_name>, 2506 AtomicNoRet<NAME # "_IDXEN_gfx12", is_return>; 2507 def _OFFEN#Rtn#_gfx12 : 2508 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_OFFEN" # Rtn, real_name>, 2509 AtomicNoRet<NAME # "_OFFEN_gfx12", is_return>; 2510 def _OFFSET#Rtn#_gfx12 : 2511 MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_OFFSET" # Rtn, real_name>, 2512 AtomicNoRet<NAME # "_OFFSET_gfx12", is_return>; 2513 } 2514} 2515 2516multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<bits<8> op, bit is_return, 2517 string real_name> : 2518 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, is_return, real_name>, 2519 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, is_return, real_name>; 2520 2521// Non-renamed gfx11/gfx12 mubuf atomic. 2522multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op> : 2523 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>, 2524 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>; 2525 2526multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> : 2527 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>, 2528 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>; 2529 2530multiclass MUBUF_Real_Atomic_gfx11_Renamed<bits<8> op, string real_name> : 2531 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, real_name>, 2532 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, real_name> { 2533 def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2534} 2535 2536multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed<bits<8> op, string real_name> : 2537 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 0, real_name>, 2538 MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 1, real_name> { 2539 def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2540} 2541 2542multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<bits<8> op, string gfx12_name, string gfx11_name> : 2543 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, gfx11_name>, 2544 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, gfx11_name>, 2545 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 0, gfx12_name>, 2546 MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 1, gfx12_name> { 2547 def : Mnem_gfx11<get_MUBUF_ps<NAME>.Mnemonic, gfx11_name>; 2548 def : Mnem_gfx12<get_MUBUF_ps<NAME>.Mnemonic, gfx12_name>; 2549 def : Mnem_gfx12<gfx11_name, gfx12_name>; 2550} 2551 2552let DecoderNamespace = "GFX11" in { 2553def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2554def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2555} 2556 2557defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x014, "buffer_load_b32">; 2558defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x015, "buffer_load_b64">; 2559defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x016, "buffer_load_b96">; 2560defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x017, "buffer_load_b128">; 2561defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x020, "buffer_load_d16_b16">; 2562defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "buffer_load_d16_format_x">; 2563defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "buffer_load_d16_format_xy">; 2564defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2565defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2566defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x023, "buffer_load_d16_hi_b16">; 2567defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2568defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x022, "buffer_load_d16_hi_i8">; 2569defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x021, "buffer_load_d16_hi_u8">; 2570defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01f, "buffer_load_d16_i8">; 2571defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01e, "buffer_load_d16_u8">; 2572defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2573defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2574defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2575defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2576defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x011, "buffer_load_i8">; 2577defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x013, "buffer_load_i16">; 2578defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x010, "buffer_load_u8">; 2579defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x012, "buffer_load_u16">; 2580defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>; 2581defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>; 2582defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>; 2583defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>; 2584defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>; 2585defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>; 2586defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x018, "buffer_store_b8">; 2587defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x019, "buffer_store_b16">; 2588defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01A, "buffer_store_b32">; 2589defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01B, "buffer_store_b64">; 2590defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01C, "buffer_store_b96">; 2591defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01D, "buffer_store_b128">; 2592defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00C, "buffer_store_d16_format_x">; 2593defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00D, "buffer_store_d16_format_xy">; 2594defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2595defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2596defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x024, "buffer_store_d16_hi_b8">; 2597defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x025, "buffer_store_d16_hi_b16">; 2598defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2599defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2600defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2601defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2602defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2603defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11_gfx12<0x056>; 2604defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x035, "buffer_atomic_add_u32">; 2605defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x043, "buffer_atomic_add_u64">; 2606defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03C, "buffer_atomic_and_b32">; 2607defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x049, "buffer_atomic_and_b64">; 2608defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2609defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2610defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2611defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">; 2612def : Mnem_gfx11_gfx12<"buffer_atomic_csub", "buffer_atomic_csub_u32">; 2613defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x040, "buffer_atomic_dec_u32">; 2614defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04D, "buffer_atomic_dec_u64">; 2615defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03F, "buffer_atomic_inc_u32">; 2616defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04C, "buffer_atomic_inc_u64">; 2617defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">; 2618defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03A, "buffer_atomic_max_i32">; 2619defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x047, "buffer_atomic_max_i64">; 2620defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03B, "buffer_atomic_max_u32">; 2621defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x048, "buffer_atomic_max_u64">; 2622defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">; 2623defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x038, "buffer_atomic_min_i32">; 2624defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x045, "buffer_atomic_min_i64">; 2625defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x039, "buffer_atomic_min_u32">; 2626defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x046, "buffer_atomic_min_u64">; 2627defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03D, "buffer_atomic_or_b32">; 2628defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04A, "buffer_atomic_or_b64">; 2629defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x036, "buffer_atomic_sub_u32">; 2630defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x044, "buffer_atomic_sub_u64">; 2631defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x033, "buffer_atomic_swap_b32">; 2632defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x041, "buffer_atomic_swap_b64">; 2633defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03E, "buffer_atomic_xor_b32">; 2634defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04B, "buffer_atomic_xor_b64">; 2635 2636//===----------------------------------------------------------------------===// 2637// MUBUF - GFX10. 2638//===----------------------------------------------------------------------===// 2639 2640let DecoderNamespace = "GFX10" in { 2641 multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> { 2642 def _BOTHEN_gfx10 : 2643 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2644 def _IDXEN_gfx10 : 2645 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2646 def _OFFEN_gfx10 : 2647 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2648 def _OFFSET_gfx10 : 2649 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2650 } 2651 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2652 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2653 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2654 } 2655 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> { 2656 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2657 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2658 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2659 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2660 2661 if !not(isTFE) then { 2662 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2663 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2664 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2665 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2666 } 2667 } 2668 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2669 def _BOTHEN_RTN_gfx10 : 2670 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2671 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2672 def _IDXEN_RTN_gfx10 : 2673 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2674 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2675 def _OFFEN_RTN_gfx10 : 2676 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2677 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2678 def _OFFSET_RTN_gfx10 : 2679 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2680 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2681 } 2682 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2683 MUBUF_Real_Atomics_RTN_gfx10<op> { 2684 def _BOTHEN_gfx10 : 2685 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2686 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2687 def _IDXEN_gfx10 : 2688 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2689 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2690 def _OFFEN_gfx10 : 2691 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2692 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2693 def _OFFSET_gfx10 : 2694 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2695 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2696 } 2697} // End DecoderNamespace = "GFX10" 2698 2699defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2700defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2701defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2702defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2703defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2704defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2705defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2706defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2707// FIXME-GFX10: Add following instructions: 2708//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2709//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2710defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2711defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2712defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2713defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2714defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2715defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2716defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2717defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2718 2719def BUFFER_GL0_INV_gfx10 : 2720 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2721def BUFFER_GL1_INV_gfx10 : 2722 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2723 2724//===----------------------------------------------------------------------===// 2725// MUBUF - GFX6, GFX7, GFX10. 2726//===----------------------------------------------------------------------===// 2727 2728let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2729 multiclass MUBUF_Real_gfx6<bits<8> op> { 2730 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2731 } 2732} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2733 2734let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2735 multiclass MUBUF_Real_gfx7<bits<8> op> { 2736 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2737 } 2738} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2739 2740let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2741 multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> { 2742 def _ADDR64_gfx6_gfx7 : 2743 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2744 def _BOTHEN_gfx6_gfx7 : 2745 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2746 def _IDXEN_gfx6_gfx7 : 2747 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2748 def _OFFEN_gfx6_gfx7 : 2749 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2750 def _OFFSET_gfx6_gfx7 : 2751 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2752 } 2753 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2754 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2755 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2756 } 2757 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> { 2758 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2759 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2760 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2761 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2762 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2763 2764 if !not(isTFE) then { 2765 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2766 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2767 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2768 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2769 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2770 } 2771 } 2772 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2773 def _ADDR64_gfx6_gfx7 : 2774 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2775 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2776 def _BOTHEN_gfx6_gfx7 : 2777 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2778 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2779 def _IDXEN_gfx6_gfx7 : 2780 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2781 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2782 def _OFFEN_gfx6_gfx7 : 2783 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2784 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2785 def _OFFSET_gfx6_gfx7 : 2786 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2787 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2788 2789 def _ADDR64_RTN_gfx6_gfx7 : 2790 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2791 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2792 def _BOTHEN_RTN_gfx6_gfx7 : 2793 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2794 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2795 def _IDXEN_RTN_gfx6_gfx7 : 2796 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2797 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2798 def _OFFEN_RTN_gfx6_gfx7 : 2799 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2800 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2801 def _OFFSET_RTN_gfx6_gfx7 : 2802 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2803 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2804 } 2805} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2806 2807multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2808 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2809 2810multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> : 2811 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>, 2812 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>; 2813 2814multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> { 2815 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>; 2816 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>; 2817} 2818 2819multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2820 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2821 2822// FIXME-GFX6: Following instructions are available only on GFX6. 2823//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2824//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2825 2826defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2827defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2828defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2829defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2830defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2831defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2832defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2833defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2834defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2835defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2836defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2837defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2838defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2839defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2840defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2841defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2842defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2843defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2844defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2845defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2846defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2847defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2848 2849defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2850defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2851defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2852defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2853defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2854defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2855defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2856defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2857defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2858defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2859defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2860defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2861defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2862defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2863defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2864defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2865defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2866defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2867defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2868defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2869defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2870defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2871defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2872defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2873defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2874defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2875defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2876defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2877defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2878// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2879defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2880defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2881defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2882 2883defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_gfx10<0x034>; 2884 2885defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2886defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2887def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2888 2889//===----------------------------------------------------------------------===// 2890// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2891//===----------------------------------------------------------------------===// 2892 2893class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2894 string real_name = ps.Mnemonic> : 2895 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2896 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2897 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2898 let Inst{31-26} = 0x3a; //encoding 2899 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2900 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2901 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2902 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2903} 2904 2905class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2906 string real_name = ps.Mnemonic> : 2907 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2908 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2909 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2910 let Inst{18-15} = op; 2911 let Inst{25-19} = format; 2912 let Inst{53} = ps.tfe; 2913 let Inst{54} = ps.offen; 2914 let Inst{55} = ps.idxen; 2915} 2916 2917class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2918 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2919 let Inst{12} = ps.offen; 2920 let Inst{13} = ps.idxen; 2921 let Inst{18-16} = op; 2922 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2923 let Inst{55} = ps.tfe; 2924} 2925 2926//===----------------------------------------------------------------------===// 2927// MTBUF - GFX11. 2928//===----------------------------------------------------------------------===// 2929 2930multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<bits<4> op, string real_name> { 2931 let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2932 def _BOTHEN_gfx11 : 2933 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2934 def _IDXEN_gfx11 : 2935 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2936 def _OFFEN_gfx11 : 2937 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2938 def _OFFSET_gfx11 : 2939 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2940 } 2941 2942 let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in { 2943 def _BOTHEN_gfx12 : 2944 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_BOTHEN"), real_name>; 2945 def _IDXEN_gfx12 : 2946 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_IDXEN"), real_name>; 2947 def _OFFEN_gfx12 : 2948 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_OFFEN"), real_name>; 2949 def _OFFSET_gfx12 : 2950 VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_OFFSET"), real_name>; 2951 } 2952} 2953 2954multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, MTBUF_Pseudo ps> 2955 : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, ps.Mnemonic>; 2956multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op> 2957 : MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2958 2959 2960class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2961 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2962multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<bits<4> op, string real_name> 2963 : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, real_name> { 2964 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2965} 2966 2967defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">; 2968defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2969defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 2970defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 2971defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>; 2972defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>; 2973defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>; 2974defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>; 2975defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00c, "tbuffer_store_d16_format_x">; 2976defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 2977defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 2978defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 2979defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>; 2980defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>; 2981defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>; 2982defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>; 2983 2984//===----------------------------------------------------------------------===// 2985// MTBUF - GFX10. 2986//===----------------------------------------------------------------------===// 2987 2988class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2989 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2990 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2991 let Inst{25-19} = format; 2992 let Inst{53} = op{3}; 2993} 2994 2995let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2996 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2997 def _BOTHEN_gfx10 : 2998 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2999 def _IDXEN_gfx10 : 3000 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3001 def _OFFEN_gfx10 : 3002 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3003 def _OFFSET_gfx10 : 3004 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3005 } 3006} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 3007 3008defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 3009defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 3010defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 3011defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 3012defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 3013defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 3014defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 3015defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 3016 3017//===----------------------------------------------------------------------===// 3018// MTBUF - GFX6, GFX7, GFX10. 3019//===----------------------------------------------------------------------===// 3020 3021class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 3022 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 3023 let Inst{15} = ps.addr64; 3024 let Inst{22-19} = dfmt; 3025 let Inst{25-23} = nfmt; 3026} 3027 3028let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 3029 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 3030 def _ADDR64_gfx6_gfx7 : 3031 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 3032 def _BOTHEN_gfx6_gfx7 : 3033 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3034 def _IDXEN_gfx6_gfx7 : 3035 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3036 def _OFFEN_gfx6_gfx7 : 3037 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3038 def _OFFSET_gfx6_gfx7 : 3039 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3040 } 3041} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 3042 3043multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 3044 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 3045 3046defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 3047defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 3048defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 3049defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 3050defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 3051defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 3052defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 3053defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 3054 3055//===----------------------------------------------------------------------===// 3056// GFX8, GFX9 (VI). 3057//===----------------------------------------------------------------------===// 3058 3059class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 3060 bit has_sccb = ps.has_sccb> : 3061 MUBUF_Real<ps>, 3062 Enc64, 3063 SIMCInstr<ps.PseudoInstr, Enc>, 3064 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 3065 !if(ps.IsAtomicRet, 1, ?))> { 3066 3067 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3068 let Inst{12} = ps.offen; 3069 let Inst{13} = ps.idxen; 3070 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3071 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3072 let Inst{16} = ps.lds; 3073 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3074 let Inst{24-18} = op; 3075 let Inst{31-26} = 0x38; //encoding 3076 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3077 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3078 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3079 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3080} 3081 3082class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 3083 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 3084 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3085 let DecoderNamespace = "GFX8"; 3086 3087 let Inst{55} = ps.tfe; 3088} 3089 3090class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 3091 bit has_sccb = ps.has_sccb> : 3092 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 3093 let AssemblerPredicate = isGFX90APlus; 3094 let DecoderNamespace = "GFX90A"; 3095 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 3096 ps.AsmOperands); 3097 3098 let Inst{55} = acc; 3099} 3100 3101class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 3102 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 3103 let AssemblerPredicate = isGFX940Plus; 3104 let DecoderNamespace = "GFX9"; 3105 let AsmString = ps.Mnemonic # ps.AsmOperands; 3106 3107 let Inst{55} = acc; 3108} 3109 3110multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps, bit isTFE = 0> { 3111 def _vi : MUBUF_Real_vi<op, ps>; 3112 3113 if !not(isTFE) then { 3114 if !not(ps.FPAtomic) then 3115 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 3116 } 3117 3118 if ps.FPAtomic then { 3119 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 3120 let SubtargetPredicate = isGFX90AOnly; 3121 let AssemblerPredicate = isGFX90AOnly; 3122 } 3123 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 3124 } 3125} 3126 3127multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> { 3128 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), isTFE>; 3129 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), isTFE>; 3130 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), isTFE>; 3131 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), isTFE>; 3132} 3133 3134multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> { 3135 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>; 3136 if hasTFE then 3137 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>; 3138} 3139 3140multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> { 3141 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 3142 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 3143 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 3144 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 3145 3146 if !not(isTFE) then { 3147 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 3148 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 3149 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 3150 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 3151 3152 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 3153 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 3154 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 3155 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 3156 3157 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 3158 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 3159 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 3160 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 3161 } 3162} 3163 3164multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 3165 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>; 3166 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>; 3167} 3168 3169class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 3170 MUBUF_Real<ps>, 3171 Enc64, 3172 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3173 let AssemblerPredicate=HasUnpackedD16VMem; 3174 let DecoderNamespace="GFX80_UNPACKED"; 3175 3176 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3177 let Inst{12} = ps.offen; 3178 let Inst{13} = ps.idxen; 3179 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3180 let Inst{16} = ps.lds; 3181 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3182 let Inst{24-18} = op; 3183 let Inst{31-26} = 0x38; //encoding 3184 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3185 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3186 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3187 let Inst{55} = ps.tfe; 3188 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3189} 3190 3191multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> { 3192 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 3193 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 3194 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 3195 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 3196} 3197 3198multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 3199 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3200 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>; 3201} 3202 3203multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 3204 MUBUF_Real_AllAddr_vi<op, 0> { 3205 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 3206 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 3207 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 3208 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 3209} 3210 3211defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 3212defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 3213defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 3214defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 3215defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 3216defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 3217defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 3218defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 3219let SubtargetPredicate = HasUnpackedD16VMem in { 3220 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 3221 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 3222 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 3223 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 3224 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 3225 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 3226 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 3227 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 3228} // End HasUnpackedD16VMem. 3229let SubtargetPredicate = HasPackedD16VMem in { 3230 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 3231 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 3232 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 3233 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 3234 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 3235 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 3236 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 3237 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 3238} // End HasPackedD16VMem. 3239defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 3240defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 3241defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 3242defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 3243defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 3244defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 3245defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 3246defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 3247defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 3248defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 3249defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 3250defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 3251defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 3252defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 3253defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 3254defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 3255 3256defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 3257defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 3258defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 3259defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 3260defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 3261defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 3262 3263defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 3264defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 3265 3266defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 3267defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 3268defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 3269defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 3270defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 3271defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 3272defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 3273defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 3274defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 3275defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 3276defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 3277defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 3278defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 3279 3280defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 3281defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 3282defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 3283defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 3284defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 3285defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 3286defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 3287defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 3288defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 3289defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 3290defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 3291defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 3292defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 3293 3294defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 3295 3296let AssemblerPredicate = isGFX8GFX9 in { 3297def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 3298def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 3299} // End AssemblerPredicate = isGFX8GFX9 3300 3301 3302defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 3303 3304let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 3305defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 3306} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 3307 3308let SubtargetPredicate = isGFX90APlus in { 3309 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 3310 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 3311 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 3312} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 3313 3314def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 3315 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 3316 let AssemblerPredicate = isGFX90AOnly; 3317 let SubtargetPredicate = isGFX90AOnly; 3318} 3319def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 3320 3321let SubtargetPredicate = isGFX940Plus in { 3322def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 3323def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 3324} 3325 3326class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 3327 MTBUF_Real<ps>, 3328 Enc64, 3329 SIMCInstr<ps.PseudoInstr, Enc> { 3330 3331 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3332 let Inst{12} = ps.offen; 3333 let Inst{13} = ps.idxen; 3334 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3335 let Inst{18-15} = op; 3336 let Inst{22-19} = dfmt; 3337 let Inst{25-23} = nfmt; 3338 let Inst{31-26} = 0x3a; //encoding 3339 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3340 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3341 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3342 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 3343 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3344 let Inst{55} = ps.tfe; 3345 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3346} 3347 3348class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 3349 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 3350 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 3351 let DecoderNamespace = "GFX8"; 3352 3353 let Inst{55} = ps.tfe; 3354} 3355 3356class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 3357 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 3358 let AssemblerPredicate = isGFX90APlus; 3359 let DecoderNamespace = "GFX90A"; 3360 let AsmString = ps.Mnemonic # ps.AsmOperands; 3361 3362 let Inst{55} = acc; 3363} 3364 3365multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 3366 def _vi : MTBUF_Real_vi<op, ps>; 3367 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 3368} 3369 3370multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 3371 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3372 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3373 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3374 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3375} 3376 3377class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 3378 MTBUF_Real<ps>, 3379 Enc64, 3380 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 3381 let AssemblerPredicate=HasUnpackedD16VMem; 3382 let DecoderNamespace="GFX80_UNPACKED"; 3383 3384 let Inst{11-0} = !if(ps.has_offset, offset, ?); 3385 let Inst{12} = ps.offen; 3386 let Inst{13} = ps.idxen; 3387 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 3388 let Inst{18-15} = op; 3389 let Inst{22-19} = dfmt; 3390 let Inst{25-23} = nfmt; 3391 let Inst{31-26} = 0x3a; //encoding 3392 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 3393 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 3394 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 3395 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 3396 let Inst{55} = ps.tfe; 3397 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 3398} 3399 3400multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 3401 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3402 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3403 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3404 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3405} 3406 3407defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 3408defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 3409defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 3410defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 3411defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 3412defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 3413defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 3414defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 3415let SubtargetPredicate = HasUnpackedD16VMem in { 3416 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 3417 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 3418 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 3419 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 3420 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 3421 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 3422 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 3423 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 3424} // End HasUnpackedD16VMem. 3425let SubtargetPredicate = HasPackedD16VMem in { 3426 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 3427 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 3428 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 3429 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 3430 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 3431 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 3432 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 3433 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 3434} // End HasUnpackedD16VMem. 3435 3436def MUBUFInfoTable : GenericTable { 3437 let FilterClass = "MUBUF_Pseudo"; 3438 let CppTypeName = "MUBUFInfo"; 3439 let Fields = [ 3440 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3441 "IsBufferInv" 3442 ]; 3443 3444 let PrimaryKey = ["Opcode"]; 3445 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3446} 3447 3448def getMUBUFInfoFromOpcode : SearchIndex { 3449 let Table = MUBUFInfoTable; 3450 let Key = ["Opcode"]; 3451} 3452 3453def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3454 let Table = MUBUFInfoTable; 3455 let Key = ["BaseOpcode", "elements"]; 3456} 3457 3458def MTBUFInfoTable : GenericTable { 3459 let FilterClass = "MTBUF_Pseudo"; 3460 let CppTypeName = "MTBUFInfo"; 3461 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3462 3463 let PrimaryKey = ["Opcode"]; 3464 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3465} 3466 3467def getMTBUFInfoFromOpcode : SearchIndex { 3468 let Table = MTBUFInfoTable; 3469 let Key = ["Opcode"]; 3470} 3471 3472def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3473 let Table = MTBUFInfoTable; 3474 let Key = ["BaseOpcode", "elements"]; 3475} 3476