1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MTBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43 44//===----------------------------------------------------------------------===// 45// BUF class (base class for MTBUF and MUBUF pseudos) 46//===----------------------------------------------------------------------===// 47 48class BUF_Pseudo <string opName, dag outs, dag ins, 49 string asmOps, list<dag> pattern=[]> : 50 InstSI<outs, ins, "", pattern>, 51 SIMCInstr<opName, SIEncodingFamily.NONE> { 52 53 let isPseudo = 1; 54 let isCodeGenOnly = 1; 55 let Size = 8; 56 let UseNamedOperandTable = 1; 57 58 string Mnemonic = opName; 59 string AsmOperands = asmOps; 60 61 Instruction Opcode = !cast<Instruction>(NAME); 62 63 64 let VM_CNT = 1; 65 let EXP_CNT = 1; 66 67 let Uses = [EXEC]; 68 let hasSideEffects = 0; 69 let SchedRW = [WriteVMEM]; 70 71 72 73 bits<1> offen = 0; 74 bits<1> idxen = 0; 75 bits<1> addr64 = 0; 76 bits<1> lds = 0; 77 bits<1> has_vdata = !not(lds); 78 bits<1> has_vaddr = 1; 79 bits<1> has_glc = 1; 80 bits<1> has_dlc = 1; 81 bits<1> glc_value = 0; // the value for glc if no such operand 82 bits<1> dlc_value = 0; // the value for dlc if no such operand 83 bits<1> has_srsrc = 1; 84 bits<1> has_soffset = 1; 85 bits<1> has_offset = 1; 86 bits<1> has_slc = 1; 87 bits<1> tfe = ?; 88 bits<4> elements = 0; 89 bits<1> has_sccb = 1; 90 bits<1> sccb_value = 0; 91 bits<1> IsBufferInv = 0; 92} 93 94 95 96//===----------------------------------------------------------------------===// 97// MTBUF classes 98//===----------------------------------------------------------------------===// 99 100class MTBUFGetBaseOpcode<string Op> { 101 string ret = !subst("FORMAT_XY", "FORMAT_X", 102 !subst("FORMAT_XYZ", "FORMAT_X", 103 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 104} 105 106 107class MTBUF_Pseudo <string opName, dag outs, dag ins, 108 string asmOps, list<dag> pattern=[]> : 109 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 110 111 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 112 let MTBUF = 1; 113 let AsmMatchConverter = "cvtMtbuf"; 114} 115 116class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 117 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 118 119 let isPseudo = 0; 120 let isCodeGenOnly = 0; 121 122 let VM_CNT = 1; 123 let EXP_CNT = 1; 124 let MTBUF = 1; 125 126 // copy relevant pseudo op flags 127 let UseNamedOperandTable = ps.UseNamedOperandTable; 128 let SubtargetPredicate = ps.SubtargetPredicate; 129 let AsmMatchConverter = ps.AsmMatchConverter; 130 let Constraints = ps.Constraints; 131 let DisableEncoding = ps.DisableEncoding; 132 let TSFlags = ps.TSFlags; 133 let SchedRW = ps.SchedRW; 134 let mayLoad = ps.mayLoad; 135 let mayStore = ps.mayStore; 136 let IsAtomicRet = ps.IsAtomicRet; 137 let IsAtomicNoRet = ps.IsAtomicNoRet; 138 139 bits<12> offset; 140 bits<5> cpol; 141 bits<7> format; 142 bits<8> vaddr; 143 bits<10> vdata; 144 bits<7> srsrc; 145 bits<8> soffset; 146 147 bits<4> dfmt = format{3-0}; 148 bits<3> nfmt = format{6-4}; 149 150 // GFX90A+ only: instruction uses AccVGPR for data 151 // Bit supersedes tfe. 152 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 153} 154 155class getMTBUFInsDA<list<RegisterClass> vdataList, 156 list<RegisterClass> vaddrList=[]> { 157 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 158 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 159 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 160 161 dag NonVaddrInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, SWZ:$swz); 162 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 163 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 164} 165 166 167class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 168 dag ret = 169 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 170 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 171 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 172 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 173 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 174 (ins)))))); 175} 176 177class getMTBUFAsmOps<int addrKind> { 178 string Pfx = 179 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 180 !if(!eq(addrKind, BUFAddrKind.OffEn), 181 "$vaddr, $srsrc,$format $soffset offen", 182 !if(!eq(addrKind, BUFAddrKind.IdxEn), 183 "$vaddr, $srsrc,$format $soffset idxen", 184 !if(!eq(addrKind, BUFAddrKind.BothEn), 185 "$vaddr, $srsrc,$format $soffset idxen offen", 186 !if(!eq(addrKind, BUFAddrKind.Addr64), 187 "$vaddr, $srsrc,$format $soffset addr64", 188 ""))))); 189 string ret = " $vdata, " # Pfx # "$offset$cpol$swz"; 190} 191 192class MTBUF_SetupAddr<int addrKind> { 193 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 194 !eq(addrKind, BUFAddrKind.BothEn)); 195 196 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 197 !eq(addrKind, BUFAddrKind.BothEn)); 198 199 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 200 201 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 202} 203 204class MTBUF_Load_Pseudo <string opName, 205 int addrKind, 206 RegisterClass vdataClass, 207 int elems, 208 list<dag> pattern=[], 209 // Workaround bug bz30254 210 int addrKindCopy = addrKind> 211 : MTBUF_Pseudo<opName, 212 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 213 getMTBUFIns<addrKindCopy>.ret, 214 getMTBUFAsmOps<addrKindCopy>.ret, 215 pattern>, 216 MTBUF_SetupAddr<addrKindCopy> { 217 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 218 let mayLoad = 1; 219 let mayStore = 0; 220 let elements = elems; 221} 222 223multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 224 int elems> { 225 226 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 227 MTBUFAddr64Table<0, NAME>; 228 229 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 230 MTBUFAddr64Table<1, NAME>; 231 232 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 233 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 234 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 235 236 let DisableWQM = 1 in { 237 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 238 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 239 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 240 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 241 } 242} 243 244class MTBUF_Store_Pseudo <string opName, 245 int addrKind, 246 RegisterClass vdataClass, 247 int elems, 248 list<dag> pattern=[], 249 // Workaround bug bz30254 250 int addrKindCopy = addrKind, 251 RegisterClass vdataClassCopy = vdataClass> 252 : MTBUF_Pseudo<opName, 253 (outs), 254 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 255 getMTBUFAsmOps<addrKindCopy>.ret, 256 pattern>, 257 MTBUF_SetupAddr<addrKindCopy> { 258 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 259 let mayLoad = 0; 260 let mayStore = 1; 261 let elements = elems; 262} 263 264multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 265 int elems> { 266 267 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 268 MTBUFAddr64Table<0, NAME>; 269 270 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 271 MTBUFAddr64Table<1, NAME>; 272 273 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 274 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 275 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 276 277 let DisableWQM = 1 in { 278 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 279 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 280 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 281 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 282 } 283} 284 285 286//===----------------------------------------------------------------------===// 287// MUBUF classes 288//===----------------------------------------------------------------------===// 289 290class MUBUFGetBaseOpcode<string Op> { 291 string ret = !subst("DWORDX2", "DWORD", 292 !subst("DWORDX3", "DWORD", 293 !subst("DWORDX4", "DWORD", Op))); 294} 295 296class MUBUF_Pseudo <string opName, dag outs, dag ins, 297 string asmOps, list<dag> pattern=[]> : 298 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 299 300 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 301 let MUBUF = 1; 302 let AsmMatchConverter = "cvtMubuf"; 303} 304 305class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 306 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 307 308 let isPseudo = 0; 309 let isCodeGenOnly = 0; 310 311 let VM_CNT = 1; 312 let EXP_CNT = 1; 313 let MUBUF = 1; 314 315 // copy relevant pseudo op flags 316 let SubtargetPredicate = ps.SubtargetPredicate; 317 let AsmMatchConverter = ps.AsmMatchConverter; 318 let OtherPredicates = ps.OtherPredicates; 319 let Constraints = ps.Constraints; 320 let DisableEncoding = ps.DisableEncoding; 321 let TSFlags = ps.TSFlags; 322 let UseNamedOperandTable = ps.UseNamedOperandTable; 323 let SchedRW = ps.SchedRW; 324 let mayLoad = ps.mayLoad; 325 let mayStore = ps.mayStore; 326 let IsAtomicRet = ps.IsAtomicRet; 327 let IsAtomicNoRet = ps.IsAtomicNoRet; 328 let VALU = ps.VALU; 329 let LGKM_CNT = ps.LGKM_CNT; 330 331 bits<12> offset; 332 bits<5> cpol; 333 bits<8> vaddr; 334 bits<10> vdata; 335 bits<7> srsrc; 336 bits<8> soffset; 337 338 // GFX90A+ only: instruction uses AccVGPR for data 339 // Bit supersedes tfe. 340 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 341} 342 343// For cache invalidation instructions. 344class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 345 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 346 347 let AsmMatchConverter = ""; 348 349 let hasSideEffects = 1; 350 let mayLoad = 0; 351 let mayStore = 0; 352 353 let IsBufferInv = 1; 354 // Set everything else to 0. 355 let offen = 0; 356 let idxen = 0; 357 let addr64 = 0; 358 let has_vdata = 0; 359 let has_vaddr = 0; 360 let has_glc = 0; 361 let has_dlc = 0; 362 let glc_value = 0; 363 let dlc_value = 0; 364 let has_srsrc = 0; 365 let has_soffset = 0; 366 let has_offset = 0; 367 let has_slc = 0; 368 let has_sccb = 0; 369 let sccb_value = 0; 370} 371 372class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> { 373 RegisterOperand tfeVDataOp = 374 !if(!eq(RC.Size, 32), AVLdSt_64, 375 !if(!eq(RC.Size, 64), AVLdSt_96, 376 !if(!eq(RC.Size, 96), AVLdSt_128, 377 !if(!eq(RC.Size, 128), AVLdSt_160, 378 RegisterOperand<VReg_1> // Invalid register. 379 )))); 380 381 RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret); 382} 383 384class getMUBUFInsDA<list<RegisterClass> vdataList, 385 list<RegisterClass> vaddrList, bit isTFE> { 386 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 387 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 388 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret; 389 390 dag NonVaddrInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol, SWZ_0:$swz); 391 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 392 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 393} 394 395class getMUBUFElements<ValueType vt> { 396 int ret = 397 !if(!eq(vt, f16), 1, 398 !if(!eq(vt, v2f16), 2, 399 !if(!eq(vt, v3f16), 3, 400 !if(!eq(vt, v4f16), 4, 401 !if(!eq(vt.Size, 32), 1, 402 !if(!eq(vt.Size, 64), 2, 403 !if(!eq(vt.Size, 96), 3, 404 !if(!eq(vt.Size, 128), 4, 0) 405 ) 406 ) 407 ) 408 ) 409 ) 410 ) 411 ); 412} 413 414class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE> { 415 dag ret = 416 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE>.ret, 417 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE>.ret, 418 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE>.ret, 419 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE>.ret, 420 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE>.ret, 421 (ins)))))); 422} 423 424class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0, bit isSwz = 0> { 425 string Vdata = !if(noVdata, " ", " $vdata, "); 426 string Lds = !if(isLds, " lds", ""); 427 string TFE = !if(isTFE, " tfe", ""); 428 string MainArgs = 429 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 430 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 431 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 432 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 433 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 434 ""))))); 435 string Offset = "$offset"; 436 string OtherArgs = "$cpol"; 437 string Swz = !if(isSwz, "$swz", ""); 438 439 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE # Swz; 440} 441 442class MUBUF_SetupAddr<int addrKind> { 443 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 444 !eq(addrKind, BUFAddrKind.BothEn)); 445 446 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 447 !eq(addrKind, BUFAddrKind.BothEn)); 448 449 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 450 451 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 452} 453 454class MUBUF_Load_Pseudo <string opName, 455 int addrKind, 456 ValueType vdata_vt, 457 bit HasTiedDest = 0, 458 bit isLds = 0, 459 bit isLdsOpc = 0, 460 bit isTFE = 0, 461 list<dag> pattern=[], 462 // Workaround bug bz30254 463 int addrKindCopy = addrKind, 464 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 465 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret> 466 : MUBUF_Pseudo<opName, 467 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 468 !con(getMUBUFIns<addrKindCopy, [], isTFE>.ret, 469 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 470 getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE, 1>.ret, 471 pattern>, 472 MUBUF_SetupAddr<addrKindCopy> { 473 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") # 474 "_" # getAddrName<addrKindCopy>.ret; 475 let AsmMatchConverter = "cvtMubuf"; 476 477 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 478 let LGKM_CNT = isLds; 479 let has_vdata = !not(!or(isLds, isLdsOpc)); 480 let mayLoad = 1; 481 let mayStore = isLds; 482 let maybeAtomic = 1; 483 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 484 let tfe = isTFE; 485 let lds = isLds; 486 let elements = getMUBUFElements<vdata_vt>.ret; 487 let VALU = isLds; 488} 489 490class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 491 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 492 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 493>; 494 495class MUBUF_Addr64_Load_Pat <Instruction inst, 496 ValueType load_vt = i32, 497 SDPatternOperator ld = null_frag> : Pat < 498 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 499 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 500>; 501 502multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 503 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 504 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 505} 506 507multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt, 508 bit TiedDest, bit isLds, bit isTFE> { 509 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 510 511 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE>, 512 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 513 514 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE>, 515 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 516 517 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 518 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 519 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 520 521 let DisableWQM = 1 in { 522 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE>; 523 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 524 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 525 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 526 } 527} 528 529multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32, 530 bit TiedDest = 0, bit isLds = 0> { 531 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0>; 532 if !not(isLds) then 533 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1>; 534} 535 536multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 537 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 538 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 539} 540 541multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 542 ValueType load_vt = i32, 543 bit TiedDest = 0, 544 bit isLds = 0, 545 bit isLdsOpc = 1> { 546 547 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 548 549 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 550 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 551 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 552 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 553} 554 555class MUBUF_Store_Pseudo <string opName, 556 int addrKind, 557 ValueType store_vt, 558 bit isTFE = 0, 559 list<dag> pattern=[], 560 // Workaround bug bz30254 561 int addrKindCopy = addrKind> 562 : MUBUF_Pseudo<opName, 563 (outs), 564 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret], isTFE>.ret, 565 getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE, 1>.ret, 566 pattern>, 567 MUBUF_SetupAddr<addrKindCopy> { 568 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") # 569 getAddrName<addrKindCopy>.ret; 570 let mayLoad = 0; 571 let mayStore = 1; 572 let maybeAtomic = 1; 573 let elements = getMUBUFElements<store_vt>.ret; 574 let tfe = isTFE; 575} 576 577multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt, 578 SDPatternOperator st, bit isTFE> { 579 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 580 581 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, 582 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 583 i16:$offset))]>, 584 MUBUFAddr64Table<0, NAME>; 585 586 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, 587 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 588 i16:$offset))]>, 589 MUBUFAddr64Table<1, NAME>; 590 591 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE>; 592 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE>; 593 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE>; 594 595 let DisableWQM = 1 in { 596 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE>; 597 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE>; 598 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE>; 599 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE>; 600 } 601} 602 603multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32, 604 SDPatternOperator st = null_frag> { 605 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, st, 0>; 606 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, null_frag, 1>; 607} 608 609class MUBUF_Pseudo_Store_Lds<string opName> 610 : MUBUF_Pseudo<opName, 611 (outs), 612 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 613 " $srsrc, $soffset$offset lds$cpol$swz"> { 614 let LGKM_CNT = 1; 615 let mayLoad = 1; 616 let mayStore = 1; 617 let maybeAtomic = 1; 618 619 let has_vdata = 0; 620 let has_vaddr = 0; 621 let lds = 1; 622 let VALU = 1; 623 624 let Uses = [EXEC, M0]; 625 let AsmMatchConverter = "cvtMubuf"; 626} 627 628class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 629 list<RegisterClass> vaddrList=[]> { 630 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 631 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 632 633 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); 634 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); 635 dag MainInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset); 636 dag CPol = !if(vdata_in, (ins CPol_GLC1:$cpol), (ins CPol_0:$cpol)); 637 638 dag ret = !con(Data, !con(MainInputs, CPol)); 639} 640 641class getMUBUFAtomicIns<int addrKind, 642 RegisterClass vdataClass, 643 bit vdata_in, 644 // Workaround bug bz30254 645 RegisterClass vdataClassCopy=vdataClass> { 646 dag ret = 647 !if(!eq(addrKind, BUFAddrKind.Offset), 648 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 649 !if(!eq(addrKind, BUFAddrKind.OffEn), 650 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 651 !if(!eq(addrKind, BUFAddrKind.IdxEn), 652 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 653 !if(!eq(addrKind, BUFAddrKind.BothEn), 654 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 655 !if(!eq(addrKind, BUFAddrKind.Addr64), 656 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 657 (ins)))))); 658} 659 660class MUBUF_Atomic_Pseudo<string opName, 661 int addrKind, 662 dag outs, 663 dag ins, 664 string asmOps, 665 list<dag> pattern=[], 666 // Workaround bug bz30254 667 int addrKindCopy = addrKind> 668 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 669 MUBUF_SetupAddr<addrKindCopy> { 670 let mayStore = 1; 671 let mayLoad = 1; 672 let hasPostISelHook = 1; 673 let hasSideEffects = 1; 674 let DisableWQM = 1; 675 let has_glc = 0; 676 let has_dlc = 0; 677 let has_sccb = 1; 678 let maybeAtomic = 1; 679 let AsmMatchConverter = "cvtMubufAtomic"; 680} 681 682class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 683 RegisterClass vdataClass, 684 list<dag> pattern=[], 685 // Workaround bug bz30254 686 int addrKindCopy = addrKind, 687 RegisterClass vdataClassCopy = vdataClass> 688 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 689 (outs), 690 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 691 getMUBUFAsmOps<addrKindCopy>.ret, 692 pattern>, 693 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 694 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 695 let glc_value = 0; 696 let dlc_value = 0; 697 let sccb_value = 0; 698 let IsAtomicNoRet = 1; 699} 700 701class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 702 RegisterClass vdataClass, 703 list<dag> pattern=[], 704 // Workaround bug bz30254 705 int addrKindCopy = addrKind, 706 RegisterClass vdataClassCopy = vdataClass, 707 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 708 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 709 (outs vdata_op:$vdata), 710 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 711 getMUBUFAsmOps<addrKindCopy>.ret, 712 pattern>, 713 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 714 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 715 let glc_value = 1; 716 let dlc_value = 0; 717 let sccb_value = 0; 718 let IsAtomicRet = 1; 719 let Constraints = "$vdata = $vdata_in"; 720 let DisableEncoding = "$vdata_in"; 721} 722 723multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 724 RegisterClass vdataClass, 725 ValueType vdataType, 726 bit isFP = isFloatType<vdataType>.ret> { 727 let FPAtomic = isFP in 728 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 729 MUBUFAddr64Table <0, NAME>; 730 731 let FPAtomic = isFP in 732 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 733 MUBUFAddr64Table <1, NAME>; 734 735 let FPAtomic = isFP in 736 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 737 738 let FPAtomic = isFP in 739 740 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 741 742 let FPAtomic = isFP in 743 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 744} 745 746multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 747 RegisterClass vdataClass, 748 ValueType vdataType, 749 SDPatternOperator atomic, 750 bit isFP = isFloatType<vdataType>.ret> { 751 let FPAtomic = isFP in 752 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 753 [(set vdataType:$vdata, 754 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 755 vdataType:$vdata_in))]>, 756 MUBUFAddr64Table <0, NAME # "_RTN">; 757 758 let FPAtomic = isFP in 759 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 760 [(set vdataType:$vdata, 761 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 762 vdataType:$vdata_in))]>, 763 MUBUFAddr64Table <1, NAME # "_RTN">; 764 765 let FPAtomic = isFP in 766 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 767 768 let FPAtomic = isFP in 769 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 770 771 let FPAtomic = isFP in 772 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 773} 774 775multiclass MUBUF_Pseudo_Atomics <string opName, 776 RegisterClass vdataClass, 777 ValueType vdataType, 778 SDPatternOperator atomic = null_frag> : 779 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 780 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 781 782 783//===----------------------------------------------------------------------===// 784// MUBUF Instructions 785//===----------------------------------------------------------------------===// 786 787defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 788 "buffer_load_format_x", f32 789>; 790defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 791 "buffer_load_format_xy", v2f32 792>; 793defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 794 "buffer_load_format_xyz", v3f32 795>; 796defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 797 "buffer_load_format_xyzw", v4f32 798>; 799defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 800 "buffer_store_format_x", f32 801>; 802defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 803 "buffer_store_format_xy", v2f32 804>; 805defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 806 "buffer_store_format_xyz", v3f32 807>; 808defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 809 "buffer_store_format_xyzw", v4f32 810>; 811 812let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 813let TiedSourceNotRead = 1 in { 814 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 815 "buffer_load_format_d16_x", i32 816 >; 817 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 818 "buffer_load_format_d16_xy", v2i32 819 >; 820 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 821 "buffer_load_format_d16_xyz", v3i32 822 >; 823 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 824 "buffer_load_format_d16_xyzw", v4i32 825 >; 826} 827 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 828 "buffer_store_format_d16_x", i32 829 >; 830 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 831 "buffer_store_format_d16_xy", v2i32 832 >; 833 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 834 "buffer_store_format_d16_xyz", v3i32 835 >; 836 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 837 "buffer_store_format_d16_xyzw", v4i32 838 >; 839} // End HasUnpackedD16VMem. 840 841let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 842let TiedSourceNotRead = 1 in { 843 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_x", f16 845 >; 846 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 847 "buffer_load_format_d16_xy", v2f16 848 >; 849 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 850 "buffer_load_format_d16_xyz", v3f16 851 >; 852 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 853 "buffer_load_format_d16_xyzw", v4f16 854 >; 855} 856 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 857 "buffer_store_format_d16_x", f16 858 >; 859 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 860 "buffer_store_format_d16_xy", v2f16 861 >; 862 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 863 "buffer_store_format_d16_xyz", v3f16 864 >; 865 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 866 "buffer_store_format_d16_xyzw", v4f16 867 >; 868} // End HasPackedD16VMem. 869 870defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 871 "buffer_load_ubyte", i32 872>; 873defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 874 "buffer_load_sbyte", i32 875>; 876defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 877 "buffer_load_ushort", i32 878>; 879defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 880 "buffer_load_sshort", i32 881>; 882defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 883 "buffer_load_dword", i32 884>; 885defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 886 "buffer_load_dwordx2", v2i32 887>; 888defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 889 "buffer_load_dwordx3", v3i32 890>; 891defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 892 "buffer_load_dwordx4", v4i32 893>; 894 895defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 896 "buffer_load_lds_b32", i32 897>; 898defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 899 "buffer_load_lds_format_x", f32 900>; 901defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 902 "buffer_load_lds_i8", i32 903>; 904defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 905 "buffer_load_lds_i16", i32 906>; 907defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 908 "buffer_load_lds_u8", i32 909>; 910defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 911 "buffer_load_lds_u16", i32 912>; 913 914defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 915defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 916defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 917defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 918defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 919defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 920defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 921defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 922defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 923defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 924defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 925defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 926defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 927defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 928 929defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 930 "buffer_store_byte", i32, truncstorei8_global 931>; 932defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 933 "buffer_store_short", i32, truncstorei16_global 934>; 935defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 936 "buffer_store_dword", i32, store_global 937>; 938defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 939 "buffer_store_dwordx2", v2i32, store_global 940>; 941defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 942 "buffer_store_dwordx3", v3i32, store_global 943>; 944defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 945 "buffer_store_dwordx4", v4i32, store_global 946>; 947defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 948 "buffer_atomic_swap", VGPR_32, i32 949>; 950defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 951 "buffer_atomic_cmpswap", VReg_64, v2i32 952>; 953defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 954 "buffer_atomic_add", VGPR_32, i32 955>; 956defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 957 "buffer_atomic_sub", VGPR_32, i32 958>; 959defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 960 "buffer_atomic_smin", VGPR_32, i32 961>; 962defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 963 "buffer_atomic_umin", VGPR_32, i32 964>; 965defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 966 "buffer_atomic_smax", VGPR_32, i32 967>; 968defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 969 "buffer_atomic_umax", VGPR_32, i32 970>; 971defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 972 "buffer_atomic_and", VGPR_32, i32 973>; 974defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 975 "buffer_atomic_or", VGPR_32, i32 976>; 977defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 978 "buffer_atomic_xor", VGPR_32, i32 979>; 980defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 981 "buffer_atomic_inc", VGPR_32, i32 982>; 983defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 984 "buffer_atomic_dec", VGPR_32, i32 985>; 986defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 987 "buffer_atomic_swap_x2", VReg_64, i64 988>; 989defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 990 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 991>; 992defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 993 "buffer_atomic_add_x2", VReg_64, i64 994>; 995defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 996 "buffer_atomic_sub_x2", VReg_64, i64 997>; 998defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 999 "buffer_atomic_smin_x2", VReg_64, i64 1000>; 1001defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1002 "buffer_atomic_umin_x2", VReg_64, i64 1003>; 1004defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1005 "buffer_atomic_smax_x2", VReg_64, i64 1006>; 1007defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1008 "buffer_atomic_umax_x2", VReg_64, i64 1009>; 1010defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1011 "buffer_atomic_and_x2", VReg_64, i64 1012>; 1013defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1014 "buffer_atomic_or_x2", VReg_64, i64 1015>; 1016defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1017 "buffer_atomic_xor_x2", VReg_64, i64 1018>; 1019defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1020 "buffer_atomic_inc_x2", VReg_64, i64 1021>; 1022defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1023 "buffer_atomic_dec_x2", VReg_64, i64 1024>; 1025 1026let SubtargetPredicate = HasGFX10_BEncoding in 1027defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1028 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1029>; 1030 1031let SubtargetPredicate = isGFX8GFX9 in { 1032def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1033} 1034 1035let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1036/* 1037defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1038defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1039*/ 1040 1041def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1042 int_amdgcn_buffer_wbinvl1_sc>; 1043} 1044 1045let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1046 1047defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1048 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1049>; 1050defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1051 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1052>; 1053defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1054 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1055>; 1056 1057} 1058 1059let SubtargetPredicate = isGFX6GFX7GFX10 in { 1060 1061defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1062 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1063>; 1064defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1065 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1066>; 1067defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1068 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1069>; 1070 1071} 1072 1073let SubtargetPredicate = HasD16LoadStore in { 1074let TiedSourceNotRead = 1 in { 1075 1076defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1077 "buffer_load_ubyte_d16", i32, 1 1078>; 1079 1080defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1081 "buffer_load_ubyte_d16_hi", i32, 1 1082>; 1083 1084defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1085 "buffer_load_sbyte_d16", i32, 1 1086>; 1087 1088defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1089 "buffer_load_sbyte_d16_hi", i32, 1 1090>; 1091 1092defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1093 "buffer_load_short_d16", i32, 1 1094>; 1095 1096defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1097 "buffer_load_short_d16_hi", i32, 1 1098>; 1099 1100defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1101 "buffer_load_format_d16_hi_x", i32 1102>; 1103} // End TiedSourceNotRead 1104 1105defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1106 "buffer_store_byte_d16_hi", i32 1107>; 1108 1109defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1110 "buffer_store_short_d16_hi", i32 1111>; 1112 1113defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1114 "buffer_store_format_d16_hi_x", i32 1115>; 1116 1117} // End HasD16LoadStore 1118 1119def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1120 int_amdgcn_buffer_wbinvl1>; 1121 1122let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1123defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1124 "buffer_atomic_add_f32", VGPR_32, f32 1125>; 1126 1127let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1128defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1129 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1130>; 1131 1132let OtherPredicates = [HasAtomicFaddRtnInsts] in 1133defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1134 "buffer_atomic_add_f32", VGPR_32, f32, null_frag 1135>; 1136 1137let OtherPredicates = [isGFX90APlus] in 1138defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1139 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag 1140>; 1141 1142//===----------------------------------------------------------------------===// 1143// MTBUF Instructions 1144//===----------------------------------------------------------------------===// 1145 1146defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1147defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1148defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1149defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1150defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1151defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1152defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1153defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1154 1155let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1156let TiedSourceNotRead = 1 in { 1157 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1158 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1159 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1160 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1161} 1162 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1163 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1164 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1165 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1166} // End HasUnpackedD16VMem. 1167 1168let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1169let TiedSourceNotRead = 1 in { 1170 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1171 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1172 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1173 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1174} 1175 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1176 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1177 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1178 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1179} // End HasPackedD16VMem. 1180 1181let SubtargetPredicate = isGFX7Plus in { 1182 1183//===----------------------------------------------------------------------===// 1184// Instruction definitions for CI and newer. 1185//===----------------------------------------------------------------------===// 1186 1187def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1188 int_amdgcn_buffer_wbinvl1_vol>; 1189 1190} // End let SubtargetPredicate = isGFX7Plus 1191 1192let SubtargetPredicate = isGFX90APlus in { 1193 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1194 let has_glc = 1; 1195 let has_sccb = 1; 1196 let InOperandList = (ins CPol_0:$cpol); 1197 let AsmOperands = "$cpol"; 1198 } 1199 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1200 let SubtargetPredicate = isGFX90AOnly; 1201 } 1202 1203 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1204 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1205 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1206} // End SubtargetPredicate = isGFX90APlus 1207 1208def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1209 let SubtargetPredicate = isGFX940Plus; 1210 let has_glc = 1; 1211 let has_sccb = 1; 1212 let InOperandList = (ins CPol_0:$cpol); 1213 let AsmOperands = "$cpol"; 1214} 1215 1216let SubtargetPredicate = isGFX10Plus in { 1217 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1218 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1219} // End SubtargetPredicate = isGFX10Plus 1220 1221//===----------------------------------------------------------------------===// 1222// MUBUF Patterns 1223//===----------------------------------------------------------------------===// 1224 1225//===----------------------------------------------------------------------===// 1226// buffer_load/store_format patterns 1227//===----------------------------------------------------------------------===// 1228 1229multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1230 string opcode, ValueType memoryVt = vt> { 1231 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1232 1233 def : GCNPat< 1234 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1235 timm:$auxiliary, 0)), 1236 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1237 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1238 >; 1239 1240 def : GCNPat< 1241 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1242 timm:$auxiliary, 0)), 1243 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1244 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1245 >; 1246 1247 def : GCNPat< 1248 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1249 timm:$auxiliary, timm)), 1250 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1251 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1252 >; 1253 1254 def : GCNPat< 1255 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1256 timm:$auxiliary, timm)), 1257 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1258 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1259 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1260 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1261 >; 1262} 1263 1264defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1265defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1266defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1267defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1268defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1269defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1270defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1271defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1272 1273defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">; 1274defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">; 1275defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">; 1276defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">; 1277 1278let SubtargetPredicate = HasUnpackedD16VMem in { 1279 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1280 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1281 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1282 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1283 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1284 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1285} // End HasUnpackedD16VMem. 1286 1287let SubtargetPredicate = HasPackedD16VMem in { 1288 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1289 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1290 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1291 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1292 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1293 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1294 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1295 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1296 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1297} // End HasPackedD16VMem. 1298 1299defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1300defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1301defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1302defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1303defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1304defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1305defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1306defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1307defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1308defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1309defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1310defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1311defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1312defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1313defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1314defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1315 1316multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1317 string opcode, ValueType memoryVt = vt> { 1318 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1319 1320 def : GCNPat< 1321 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1322 timm:$auxiliary, 0), 1323 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1324 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1325 >; 1326 1327 def : GCNPat< 1328 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1329 timm:$auxiliary, 0), 1330 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1331 (as_i16timm $offset), (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1332 >; 1333 1334 def : GCNPat< 1335 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1336 timm:$auxiliary, timm), 1337 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1338 (as_i16timm $offset), (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1339 >; 1340 1341 def : GCNPat< 1342 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1343 timm:$auxiliary, timm), 1344 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1345 getVregSrcForVT<vt>.ret:$vdata, 1346 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1347 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1348 (extract_swz $auxiliary)) 1349 >; 1350} 1351 1352defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1353defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1354defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1355defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1356defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1357defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1358defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1359defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1360 1361let SubtargetPredicate = HasUnpackedD16VMem in { 1362 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1363 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1364 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1365 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1366 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1367 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1368} // End HasUnpackedD16VMem. 1369 1370let SubtargetPredicate = HasPackedD16VMem in { 1371 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1372 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1373 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1374 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1375 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1376 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1377 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1378 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1379 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1380} // End HasPackedD16VMem. 1381 1382defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1383defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1384defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1385defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1386defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1387defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1388defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1389defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1390defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1391defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1392defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1393defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1394defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1395defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1396 1397//===----------------------------------------------------------------------===// 1398// buffer_atomic patterns 1399//===----------------------------------------------------------------------===// 1400 1401multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1402 foreach RtnMode = ["ret", "noret"] in { 1403 1404 defvar Op = !cast<SDPatternOperator>(OpPrefix 1405 # !if(!eq(RtnMode, "ret"), "", "_noret") 1406 # !if(isIntr, "", "_" # vt.Size)); 1407 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1408 1409 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1410 def : GCNPat< 1411 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1412 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1413 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1414 >; 1415 1416 def : GCNPat< 1417 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1418 vt:$vdata_in)), 1419 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1420 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1421 >; 1422 } // end let AddedComplexity 1423 1424 } // end foreach RtnMode 1425} 1426 1427multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1428 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1429} 1430 1431multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1432 foreach RtnMode = ["ret", "noret"] in { 1433 1434 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1435 # !if(!eq(RtnMode, "ret"), "", "_noret") 1436 # "_" # vt.Size); 1437 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1438 1439 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1440 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1441 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1442 offset:$offset); 1443 def : GCNPat< 1444 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1445 !if(!eq(RtnMode, "ret"), 1446 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1447 !if(!eq(vt, i32), sub0, sub0_sub1)), 1448 OffsetResDag) 1449 >; 1450 1451 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1452 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1453 SCSrc_b32:$soffset, offset:$offset); 1454 def : GCNPat< 1455 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1456 data_vt:$vdata_in)), 1457 !if(!eq(RtnMode, "ret"), 1458 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1459 !if(!eq(vt, i32), sub0, sub0_sub1)), 1460 Addr64ResDag) 1461 >; 1462 } // end let AddedComplexity 1463 1464 } // end foreach RtnMode 1465} 1466 1467foreach Ty = [i32, i64] in { 1468 1469defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1470 1471defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1472defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1473defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1474defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1475defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1476defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1477defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1478defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1479defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1480defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1481defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1482defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1483 1484} // end foreach Ty 1485 1486defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1487defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1488 1489multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1490 list<string> RtnModes = ["ret", "noret"]> { 1491 foreach RtnMode = RtnModes in { 1492 1493 defvar Op = !cast<SDPatternOperator>(OpPrefix 1494 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1495 1496 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1497 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1498 (set_glc $cachepolicy), (timm:$cachepolicy)); 1499 1500 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1501 def : GCNPat< 1502 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1503 timm:$offset, timm:$cachepolicy, 0)), 1504 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1505 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1506 (as_i16timm $offset), CachePolicy) 1507 >; 1508 1509 def : GCNPat< 1510 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1511 timm:$offset, timm:$cachepolicy, timm)), 1512 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1513 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1514 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1515 >; 1516 1517 def : GCNPat< 1518 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1519 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1520 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1521 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1522 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1523 >; 1524 1525 def : GCNPat< 1526 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1527 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1528 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1529 getVregSrcForVT<vt>.ret:$vdata_in, 1530 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1531 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1532 >; 1533 } // end let AddedComplexity 1534 1535 } // end foreach RtnMode 1536} 1537 1538defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1539defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1540defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1541defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1542defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1543defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1544defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1545defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1546defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1547defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1548defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1549defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1550defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1551defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1552defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1553defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1554defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1555defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1556defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1557defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1558defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1559defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1560defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1561defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1562defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1563defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1564 1565let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1566 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1567 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1568} 1569let SubtargetPredicate = isGFX6GFX7GFX10 in { 1570 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1571 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1572} 1573 1574class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1575 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1576 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> { 1577 let HasNoUse = true; 1578} 1579 1580multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1581 string opcode> { 1582 def : GCNPat< 1583 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1584 0, i32:$soffset, timm:$offset, 1585 timm:$cachepolicy, 0), 1586 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1587 (as_i16timm $offset), timm:$cachepolicy) 1588 >; 1589 1590 def : GCNPat< 1591 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1592 0, i32:$soffset, timm:$offset, 1593 timm:$cachepolicy, timm), 1594 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1595 (as_i16timm $offset), timm:$cachepolicy) 1596 >; 1597 1598 def : GCNPat< 1599 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1600 i32:$voffset, i32:$soffset, timm:$offset, 1601 timm:$cachepolicy, 0), 1602 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1603 (as_i16timm $offset), timm:$cachepolicy) 1604 >; 1605 1606 def : GCNPat< 1607 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1608 i32:$voffset, i32:$soffset, timm:$offset, 1609 timm:$cachepolicy, timm), 1610 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1611 getVregSrcForVT<vt>.ret:$vdata_in, 1612 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1613 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), timm:$cachepolicy) 1614 >; 1615} 1616 1617let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1618defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>; 1619 1620let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1621defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>; 1622 1623let SubtargetPredicate = HasAtomicFaddRtnInsts in 1624defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>; 1625 1626let SubtargetPredicate = isGFX90APlus in { 1627 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>; 1628 1629 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1630 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1631 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1632} // End SubtargetPredicate = isGFX90APlus 1633 1634foreach RtnMode = ["ret", "noret"] in { 1635 1636defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1637 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1638defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1639defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1640 (timm:$cachepolicy)); 1641 1642defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1643 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1644 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1645def : GCNPat< 1646 (Op 1647 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1648 timm:$offset, timm:$cachepolicy, 0), 1649 !if(!eq(RtnMode, "ret"), 1650 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1651 OffsetResDag) 1652>; 1653 1654defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1655 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1656 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1657 CachePolicy); 1658def : GCNPat< 1659 (Op 1660 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1661 0, i32:$soffset, timm:$offset, 1662 timm:$cachepolicy, timm), 1663 !if(!eq(RtnMode, "ret"), 1664 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1665 IdxenResDag) 1666>; 1667 1668defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1669 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1670 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1671 CachePolicy); 1672def : GCNPat< 1673 (Op 1674 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1675 i32:$voffset, i32:$soffset, timm:$offset, 1676 timm:$cachepolicy, 0), 1677 !if(!eq(RtnMode, "ret"), 1678 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1679 OffenResDag) 1680>; 1681 1682defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1683 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1684 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1685 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1686def : GCNPat< 1687 (Op 1688 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1689 i32:$voffset, i32:$soffset, timm:$offset, 1690 timm:$cachepolicy, timm), 1691 !if(!eq(RtnMode, "ret"), 1692 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1693 BothenResDag) 1694>; 1695 1696} // end foreach RtnMode 1697 1698class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1699 PatFrag constant_ld> : GCNPat < 1700 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1701 i16:$offset))), 1702 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1703 >; 1704 1705multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1706 ValueType vt, PatFrag atomic_ld> { 1707 def : GCNPat < 1708 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1709 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1710 >; 1711 1712 def : GCNPat < 1713 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1714 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1715 >; 1716} 1717 1718let SubtargetPredicate = isGFX6GFX7 in { 1719def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1720def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1721def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1722def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1723def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1724def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1725 1726defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1727defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1728} // End SubtargetPredicate = isGFX6GFX7 1729 1730multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1731 PatFrag ld> { 1732 1733 def : GCNPat < 1734 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1735 (Instr_OFFSET $srsrc, $soffset, $offset) 1736 >; 1737} 1738 1739let OtherPredicates = [Has16BitInsts] in { 1740 1741defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1742defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1743defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1744defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1745defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1746defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1747 1748defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1749 1750} // End OtherPredicates = [Has16BitInsts] 1751 1752multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1753 MUBUF_Pseudo InstrOffset, 1754 ValueType vt, PatFrag ld> { 1755 def : GCNPat < 1756 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1757 i32:$soffset, u16imm:$offset))), 1758 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0) 1759 >; 1760 1761 def : GCNPat < 1762 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1763 (InstrOffset $srsrc, $soffset, $offset, 0, 0) 1764 >; 1765} 1766 1767// XXX - Is it possible to have a complex pattern in a PatFrag? 1768multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1769 MUBUF_Pseudo InstrOffset, 1770 ValueType vt, PatFrag ld_frag> { 1771 def : GCNPat < 1772 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1773 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1774 >; 1775 1776 def : GCNPat < 1777 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1778 (InstrOffset $srsrc, $soffset, $offset, $in) 1779 >; 1780} 1781 1782let OtherPredicates = [DisableFlatScratch] in { 1783defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1784defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1785defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1786defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1787defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1788defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1789defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1790defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1791defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1792defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1793 1794foreach vt = Reg32Types.types in { 1795defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1796} 1797defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1798defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1799defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1800 1801let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1802defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1803defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1804defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1805defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1806defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1807defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1808 1809defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1810defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1811defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1812defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1813defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1814defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1815} 1816 1817} // End OtherPredicates = [DisableFlatScratch] 1818 1819multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1820 ValueType vt, PatFrag atomic_st> { 1821 // Store follows atomic op convention so address is first 1822 def : GCNPat < 1823 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1824 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1825 >; 1826 1827 def : GCNPat < 1828 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1829 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1830 >; 1831} 1832let SubtargetPredicate = isGFX6GFX7 in { 1833defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1834defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1835defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1836defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1837defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1838defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1839} // End Predicates = isGFX6GFX7 1840 1841 1842multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1843 PatFrag st> { 1844 1845 def : GCNPat < 1846 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1847 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1848 >; 1849} 1850 1851defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1852defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1853 1854multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1855 MUBUF_Pseudo InstrOffset, 1856 ValueType vt, PatFrag st, 1857 RegisterClass rc = VGPR_32> { 1858 def : GCNPat < 1859 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1860 i32:$soffset, u16imm:$offset)), 1861 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) 1862 >; 1863 1864 def : GCNPat < 1865 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1866 u16imm:$offset)), 1867 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0) 1868 >; 1869} 1870 1871let OtherPredicates = [DisableFlatScratch] in { 1872defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1873defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1874defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1875defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1876 1877foreach vt = Reg32Types.types in { 1878defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1879} 1880 1881defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1882defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1883defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1884 1885 1886let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1887 // Hiding the extract high pattern in the PatFrag seems to not 1888 // automatically increase the complexity. 1889let AddedComplexity = 1 in { 1890defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1891defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1892} 1893} 1894} // End OtherPredicates = [DisableFlatScratch] 1895 1896//===----------------------------------------------------------------------===// 1897// MTBUF Patterns 1898//===----------------------------------------------------------------------===// 1899 1900//===----------------------------------------------------------------------===// 1901// tbuffer_load/store_format patterns 1902//===----------------------------------------------------------------------===// 1903 1904multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1905 string opcode, ValueType memoryVt = vt> { 1906 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1907 1908 def : GCNPat< 1909 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1910 timm:$format, timm:$auxiliary, 0)), 1911 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1912 (as_i8timm $format), 1913 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1914 >; 1915 1916 def : GCNPat< 1917 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1918 timm:$format, timm:$auxiliary, timm)), 1919 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1920 (as_i8timm $format), 1921 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1922 >; 1923 1924 def : GCNPat< 1925 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1926 timm:$format, timm:$auxiliary, 0)), 1927 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1928 (as_i8timm $format), 1929 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1930 >; 1931 1932 def : GCNPat< 1933 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1934 timm:$format, timm:$auxiliary, timm)), 1935 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1936 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1937 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1938 (as_i8timm $format), 1939 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1940 >; 1941} 1942 1943defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1944defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1945defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1946defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1947defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1948defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1949defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1950defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1951 1952let SubtargetPredicate = HasUnpackedD16VMem in { 1953 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1954 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1955 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1956 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1957 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1958} // End HasUnpackedD16VMem. 1959 1960let SubtargetPredicate = HasPackedD16VMem in { 1961 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1962 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1963 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1964 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1965 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1966} // End HasPackedD16VMem. 1967 1968multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1969 string opcode, ValueType memoryVt = vt> { 1970 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1971 1972 def : GCNPat< 1973 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1974 timm:$format, timm:$auxiliary, 0), 1975 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1976 (as_i16timm $offset), (as_i8timm $format), 1977 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1978 >; 1979 1980 def : GCNPat< 1981 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1982 timm:$format, timm:$auxiliary, timm), 1983 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1984 (as_i16timm $offset), (as_i8timm $format), 1985 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1986 >; 1987 1988 def : GCNPat< 1989 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1990 timm:$format, timm:$auxiliary, 0), 1991 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1992 (as_i16timm $offset), (as_i8timm $format), 1993 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1994 >; 1995 1996 def : GCNPat< 1997 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1998 timm:$offset, timm:$format, timm:$auxiliary, timm), 1999 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2000 getVregSrcForVT<vt>.ret:$vdata, 2001 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2002 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 2003 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 2004 >; 2005} 2006 2007defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2008defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2009defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2010defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2011defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2012defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2013defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2014defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2015 2016let SubtargetPredicate = HasUnpackedD16VMem in { 2017 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2018 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2019 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2020 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2021 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2022} // End HasUnpackedD16VMem. 2023 2024let SubtargetPredicate = HasPackedD16VMem in { 2025 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2026 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2027 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2028 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2029 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2030} // End HasPackedD16VMem. 2031 2032//===----------------------------------------------------------------------===// 2033// Target-specific instruction encodings. 2034//===----------------------------------------------------------------------===// 2035 2036//===----------------------------------------------------------------------===// 2037// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2038//===----------------------------------------------------------------------===// 2039 2040class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2041 string real_name = ps.Mnemonic> : 2042 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2043 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2044 let Inst{31-26} = 0x38; 2045 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2046 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2047 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2048 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2049} 2050 2051class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2052 string real_name = ps.Mnemonic> : 2053 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2054 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2055 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2056 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2057 let Inst{25-18} = op; 2058 let Inst{53} = ps.tfe; 2059 let Inst{54} = ps.offen; 2060 let Inst{55} = ps.idxen; 2061} 2062 2063class Base_MUBUF_Real_Atomic_gfx11<bits<8> op, MUBUF_Pseudo ps, 2064 string real_name> : 2065 MUBUF_Real_gfx11<op, ps, real_name> { 2066 let Inst{13} = cpol{CPolBit.DLC}; 2067} 2068 2069class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2070 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2071 let Inst{12} = ps.offen; 2072 let Inst{13} = ps.idxen; 2073 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2074 let Inst{16} = ps.lds; 2075 let Inst{24-18} = op; 2076 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2077 let Inst{55} = ps.tfe; 2078} 2079 2080class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2081 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2082 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2083 let Inst{25} = op{7}; 2084} 2085 2086class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2087 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2088 let Inst{15} = ps.addr64; 2089} 2090 2091//===----------------------------------------------------------------------===// 2092// MUBUF - GFX11. 2093//===----------------------------------------------------------------------===// 2094 2095// Shortcut to default Mnemonic from MUBUF_Pseudo. Hides the cast to the 2096// specific pseudo (bothen in this case) since any of them will work. 2097class get_MUBUF_ps<string name> { 2098 string Mnemonic = !cast<MUBUF_Pseudo>(name # "_BOTHEN").Mnemonic; 2099} 2100 2101// gfx11 instruction that accept both old and new assembler name. 2102class Pre_gfx11_MUBUF_Name <string mnemonic, string real_name> : 2103 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>; 2104 2105class MUBUF_Real_gfx11_impl<bits<8> op, string ps_name, string real_name> : 2106 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2107let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2108multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<bits<8> op, string real_name> { 2109 def _BOTHEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_BOTHEN", real_name>; 2110 def _IDXEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_IDXEN", real_name>; 2111 def _OFFEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFEN", real_name>; 2112 def _OFFSET_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFSET", real_name>; 2113} 2114 2115multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name, 2116 bit hasTFE = 1> { 2117 defm NAME : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2118 if hasTFE then 2119 defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2120} 2121 2122// Non-renamed, non-atomic gfx11 mubuf instructions. 2123multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> : 2124 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>; 2125 2126multiclass MUBUF_Real_AllAddr_gfx11_Renamed<bits<8> op, string real_name> : 2127 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2128 def : Pre_gfx11_MUBUF_Name<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2129} 2130 2131class MUBUF_Real_Atomic_gfx11_impl<bits<8> op, string ps_name, 2132 string real_name> : 2133 Base_MUBUF_Real_Atomic_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2134let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2135multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return, 2136 string real_name> { 2137 defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); 2138 def _BOTHEN#Rtn#_gfx11 : 2139 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_BOTHEN" # Rtn, real_name>, 2140 AtomicNoRet<NAME # "_BOTHEN_gfx11", is_return>; 2141 def _IDXEN#Rtn#_gfx11 : 2142 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_IDXEN" # Rtn, real_name>, 2143 AtomicNoRet<NAME # "_IDXEN_gfx11", is_return>; 2144 def _OFFEN#Rtn#_gfx11 : 2145 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFEN" # Rtn, real_name>, 2146 AtomicNoRet<NAME # "_OFFEN_gfx11", is_return>; 2147 def _OFFSET#Rtn#_gfx11 : 2148 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFSET" # Rtn, real_name>, 2149 AtomicNoRet<NAME # "_OFFSET_gfx11", is_return>; 2150} 2151 2152// Non-renamed gfx11 mubuf atomic. 2153multiclass MUBUF_Real_Atomic_gfx11<bits<8> op> : 2154 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>, 2155 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>; 2156 2157multiclass MUBUF_Real_Atomic_gfx11_Renamed<bits<8> op, string real_name> : 2158 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, real_name>, 2159 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, real_name> { 2160 def : Pre_gfx11_MUBUF_Name<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2161} 2162 2163let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2164def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2165def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2166} 2167 2168defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x014, "buffer_load_b32">; 2169defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x015, "buffer_load_b64">; 2170defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x016, "buffer_load_b96">; 2171defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x017, "buffer_load_b128">; 2172defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x020, "buffer_load_d16_b16">; 2173defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x008, "buffer_load_d16_format_x">; 2174defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x009, "buffer_load_d16_format_xy">; 2175defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2176defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2177defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x023, "buffer_load_d16_hi_b16">; 2178defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2179defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x022, "buffer_load_d16_hi_i8">; 2180defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x021, "buffer_load_d16_hi_u8">; 2181defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01f, "buffer_load_d16_i8">; 2182defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01e, "buffer_load_d16_u8">; 2183defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x000>; 2184defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x001>; 2185defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x002>; 2186defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x003>; 2187defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x011, "buffer_load_i8">; 2188defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x013, "buffer_load_i16">; 2189defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x010, "buffer_load_u8">; 2190defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x012, "buffer_load_u16">; 2191defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>; 2192defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>; 2193defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>; 2194defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>; 2195defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>; 2196defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>; 2197defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x018, "buffer_store_b8">; 2198defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x019, "buffer_store_b16">; 2199defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x01A, "buffer_store_b32">; 2200defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01B, "buffer_store_b64">; 2201defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01C, "buffer_store_b96">; 2202defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01D, "buffer_store_b128">; 2203defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x00C, "buffer_store_d16_format_x">; 2204defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x00D, "buffer_store_d16_format_xy">; 2205defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2206defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2207defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x024, "buffer_store_d16_hi_b8">; 2208defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x025, "buffer_store_d16_hi_b16">; 2209defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2210defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x004>; 2211defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x005>; 2212defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x006>; 2213defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x007>; 2214defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11<0x056>; 2215defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_Renamed<0x035, "buffer_atomic_add_u32">; 2216defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x043, "buffer_atomic_add_u64">; 2217defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_Renamed<0x03C, "buffer_atomic_and_b32">; 2218defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x049, "buffer_atomic_and_b64">; 2219defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2220defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2221defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2222defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_Renamed_impl<0x037, 1, "buffer_atomic_csub_u32">; 2223def : Pre_gfx11_MUBUF_Name<"buffer_atomic_csub", "buffer_atomic_csub_u32">; 2224defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">; 2225defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">; 2226defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_Renamed<0x03F, "buffer_atomic_inc_u32">; 2227defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04C, "buffer_atomic_inc_u64">; 2228defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_Renamed<0x052, "buffer_atomic_max_f32">; 2229defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_Renamed<0x03A, "buffer_atomic_max_i32">; 2230defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x047, "buffer_atomic_max_i64">; 2231defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_Renamed<0x03B, "buffer_atomic_max_u32">; 2232defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x048, "buffer_atomic_max_u64">; 2233defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_Renamed<0x051, "buffer_atomic_min_f32">; 2234defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_Renamed<0x038, "buffer_atomic_min_i32">; 2235defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x045, "buffer_atomic_min_i64">; 2236defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_Renamed<0x039, "buffer_atomic_min_u32">; 2237defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x046, "buffer_atomic_min_u64">; 2238defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_Renamed<0x03D, "buffer_atomic_or_b32">; 2239defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04A, "buffer_atomic_or_b64">; 2240defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_Renamed<0x036, "buffer_atomic_sub_u32">; 2241defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x044, "buffer_atomic_sub_u64">; 2242defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x033, "buffer_atomic_swap_b32">; 2243defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x041, "buffer_atomic_swap_b64">; 2244defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_Renamed<0x03E, "buffer_atomic_xor_b32">; 2245defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04B, "buffer_atomic_xor_b64">; 2246 2247//===----------------------------------------------------------------------===// 2248// MUBUF - GFX10. 2249//===----------------------------------------------------------------------===// 2250 2251let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2252 multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> { 2253 def _BOTHEN_gfx10 : 2254 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2255 def _IDXEN_gfx10 : 2256 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2257 def _OFFEN_gfx10 : 2258 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2259 def _OFFSET_gfx10 : 2260 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2261 } 2262 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2263 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2264 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2265 } 2266 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> { 2267 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2268 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2269 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2270 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2271 2272 if !not(isTFE) then { 2273 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2274 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2275 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2276 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2277 } 2278 } 2279 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2280 def _BOTHEN_RTN_gfx10 : 2281 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2282 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2283 def _IDXEN_RTN_gfx10 : 2284 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2285 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2286 def _OFFEN_RTN_gfx10 : 2287 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2288 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2289 def _OFFSET_RTN_gfx10 : 2290 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2291 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2292 } 2293 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2294 MUBUF_Real_Atomics_RTN_gfx10<op> { 2295 def _BOTHEN_gfx10 : 2296 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2297 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2298 def _IDXEN_gfx10 : 2299 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2300 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2301 def _OFFEN_gfx10 : 2302 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2303 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2304 def _OFFSET_gfx10 : 2305 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2306 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2307 } 2308} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2309 2310defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2311defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2312defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2313defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2314defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2315defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2316defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2317defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2318// FIXME-GFX10: Add following instructions: 2319//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2320//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2321defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2322defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2323defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2324defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2325defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2326defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2327defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2328defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2329 2330def BUFFER_GL0_INV_gfx10 : 2331 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2332def BUFFER_GL1_INV_gfx10 : 2333 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2334 2335//===----------------------------------------------------------------------===// 2336// MUBUF - GFX6, GFX7, GFX10. 2337//===----------------------------------------------------------------------===// 2338 2339let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2340 multiclass MUBUF_Real_gfx6<bits<8> op> { 2341 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2342 } 2343} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2344 2345let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2346 multiclass MUBUF_Real_gfx7<bits<8> op> { 2347 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2348 } 2349} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2350 2351let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2352 multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> { 2353 def _ADDR64_gfx6_gfx7 : 2354 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2355 def _BOTHEN_gfx6_gfx7 : 2356 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2357 def _IDXEN_gfx6_gfx7 : 2358 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2359 def _OFFEN_gfx6_gfx7 : 2360 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2361 def _OFFSET_gfx6_gfx7 : 2362 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2363 } 2364 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2365 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2366 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2367 } 2368 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> { 2369 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2370 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2371 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2372 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2373 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2374 2375 if !not(isTFE) then { 2376 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2377 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2378 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2379 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2380 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2381 } 2382 } 2383 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2384 def _ADDR64_gfx6_gfx7 : 2385 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2386 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2387 def _BOTHEN_gfx6_gfx7 : 2388 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2389 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2390 def _IDXEN_gfx6_gfx7 : 2391 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2392 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2393 def _OFFEN_gfx6_gfx7 : 2394 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2395 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2396 def _OFFSET_gfx6_gfx7 : 2397 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2398 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2399 2400 def _ADDR64_RTN_gfx6_gfx7 : 2401 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2402 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2403 def _BOTHEN_RTN_gfx6_gfx7 : 2404 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2405 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2406 def _IDXEN_RTN_gfx6_gfx7 : 2407 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2408 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2409 def _OFFEN_RTN_gfx6_gfx7 : 2410 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2411 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2412 def _OFFSET_RTN_gfx6_gfx7 : 2413 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2414 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2415 } 2416} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2417 2418multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2419 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2420 2421multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> : 2422 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>, 2423 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>; 2424 2425multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> { 2426 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>; 2427 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>; 2428} 2429 2430multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2431 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2432 2433// FIXME-GFX6: Following instructions are available only on GFX6. 2434//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2435//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2436 2437defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2438defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2439defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2440defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2441defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2442defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2443defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2444defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2445defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2446defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2447defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2448defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2449defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2450defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2451defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2452defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2453defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2454defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2455defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2456defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2457defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2458defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2459 2460defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2461defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2462defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2463defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2464defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2465defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2466defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2467defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2468defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2469defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2470defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2471defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2472defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2473defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2474defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2475defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2476defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2477defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2478defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2479defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2480defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2481defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2482defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2483defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2484defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2485defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2486defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2487defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2488defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2489// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2490defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2491defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2492defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2493 2494defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2495 2496defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2497defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2498def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2499 2500//===----------------------------------------------------------------------===// 2501// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2502//===----------------------------------------------------------------------===// 2503 2504class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2505 string real_name = ps.Mnemonic> : 2506 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2507 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2508 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2509 let Inst{31-26} = 0x3a; //encoding 2510 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2511 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2512 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2513 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2514} 2515 2516class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2517 string real_name = ps.Mnemonic> : 2518 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2519 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2520 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2521 let Inst{18-15} = op; 2522 let Inst{25-19} = format; 2523 let Inst{53} = ps.tfe; 2524 let Inst{54} = ps.offen; 2525 let Inst{55} = ps.idxen; 2526} 2527 2528class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2529 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2530 let Inst{12} = ps.offen; 2531 let Inst{13} = ps.idxen; 2532 let Inst{18-16} = op; 2533 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2534 let Inst{55} = ps.tfe; 2535} 2536 2537//===----------------------------------------------------------------------===// 2538// MTBUF - GFX11. 2539//===----------------------------------------------------------------------===// 2540 2541let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2542multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> { 2543 def _BOTHEN_gfx11 : 2544 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2545 def _IDXEN_gfx11 : 2546 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2547 def _OFFEN_gfx11 : 2548 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2549 def _OFFSET_gfx11 : 2550 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2551} 2552 2553multiclass MTBUF_Real_AllAddr_gfx11_Impl<bits<4> op, MTBUF_Pseudo ps> 2554 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2555multiclass MTBUF_Real_AllAddr_gfx11<bits<4> op> 2556 : MTBUF_Real_AllAddr_gfx11_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2557 2558 2559class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2560 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2561multiclass MTBUF_Real_AllAddr_gfx11_Renamed<bits<4> op, string real_name> 2562 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2563 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2564} 2565 2566defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x008, "tbuffer_load_d16_format_x">; 2567defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2568defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 2569defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 2570defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x000>; 2571defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x001>; 2572defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x002>; 2573defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x003>; 2574defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x00c, "tbuffer_store_d16_format_x">; 2575defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 2576defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 2577defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 2578defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x004>; 2579defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x005>; 2580defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x006>; 2581defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x007>; 2582 2583//===----------------------------------------------------------------------===// 2584// MTBUF - GFX10. 2585//===----------------------------------------------------------------------===// 2586 2587class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2588 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2589 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2590 let Inst{25-19} = format; 2591 let Inst{53} = op{3}; 2592} 2593 2594let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2595 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2596 def _BOTHEN_gfx10 : 2597 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2598 def _IDXEN_gfx10 : 2599 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2600 def _OFFEN_gfx10 : 2601 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2602 def _OFFSET_gfx10 : 2603 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2604 } 2605} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2606 2607defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2608defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2609defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2610defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2611defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2612defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2613defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2614defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2615 2616//===----------------------------------------------------------------------===// 2617// MTBUF - GFX6, GFX7, GFX10. 2618//===----------------------------------------------------------------------===// 2619 2620class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2621 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2622 let Inst{15} = ps.addr64; 2623 let Inst{22-19} = dfmt; 2624 let Inst{25-23} = nfmt; 2625} 2626 2627let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2628 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2629 def _ADDR64_gfx6_gfx7 : 2630 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2631 def _BOTHEN_gfx6_gfx7 : 2632 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2633 def _IDXEN_gfx6_gfx7 : 2634 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2635 def _OFFEN_gfx6_gfx7 : 2636 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2637 def _OFFSET_gfx6_gfx7 : 2638 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2639 } 2640} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2641 2642multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2643 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2644 2645defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2646defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2647defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2648defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2649defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2650defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2651defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2652defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2653 2654//===----------------------------------------------------------------------===// 2655// GFX8, GFX9 (VI). 2656//===----------------------------------------------------------------------===// 2657 2658class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2659 bit has_sccb = ps.has_sccb> : 2660 MUBUF_Real<ps>, 2661 Enc64, 2662 SIMCInstr<ps.PseudoInstr, Enc>, 2663 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2664 !if(ps.IsAtomicRet, 1, ?))> { 2665 2666 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2667 let Inst{12} = ps.offen; 2668 let Inst{13} = ps.idxen; 2669 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2670 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2671 let Inst{16} = ps.lds; 2672 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2673 let Inst{24-18} = op; 2674 let Inst{31-26} = 0x38; //encoding 2675 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2676 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2677 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2678 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2679} 2680 2681class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2682 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2683 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2684 let DecoderNamespace = "GFX8"; 2685 2686 let Inst{55} = ps.tfe; 2687} 2688 2689class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2690 bit has_sccb = ps.has_sccb> : 2691 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2692 let AssemblerPredicate = isGFX90APlus; 2693 let DecoderNamespace = "GFX90A"; 2694 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2695 ps.AsmOperands); 2696 2697 let Inst{55} = acc; 2698} 2699 2700class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2701 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2702 let AssemblerPredicate = isGFX940Plus; 2703 let DecoderNamespace = "GFX9"; 2704 let AsmString = ps.Mnemonic # ps.AsmOperands; 2705 2706 let Inst{55} = acc; 2707} 2708 2709multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps, bit isTFE = 0> { 2710 def _vi : MUBUF_Real_vi<op, ps>; 2711 2712 if !not(isTFE) then { 2713 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2714 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2715 } 2716 2717 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2718 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2719 let SubtargetPredicate = isGFX90AOnly; 2720 let AssemblerPredicate = isGFX90AOnly; 2721 } 2722 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2723 } 2724} 2725 2726multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> { 2727 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), isTFE>; 2728 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), isTFE>; 2729 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), isTFE>; 2730 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), isTFE>; 2731} 2732 2733multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> { 2734 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>; 2735 if hasTFE then 2736 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>; 2737} 2738 2739multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> { 2740 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2741 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2742 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2743 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2744 2745 if !not(isTFE) then { 2746 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2747 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2748 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2749 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2750 2751 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2752 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2753 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2754 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2755 2756 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2757 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2758 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2759 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2760 } 2761} 2762 2763multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2764 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>; 2765 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>; 2766} 2767 2768class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2769 MUBUF_Real<ps>, 2770 Enc64, 2771 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2772 let AssemblerPredicate=HasUnpackedD16VMem; 2773 let DecoderNamespace="GFX80_UNPACKED"; 2774 2775 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2776 let Inst{12} = ps.offen; 2777 let Inst{13} = ps.idxen; 2778 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2779 let Inst{16} = ps.lds; 2780 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2781 let Inst{24-18} = op; 2782 let Inst{31-26} = 0x38; //encoding 2783 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2784 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2785 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2786 let Inst{55} = ps.tfe; 2787 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2788} 2789 2790multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> { 2791 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2792 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2793 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2794 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2795} 2796 2797multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2798 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>; 2799 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>; 2800} 2801 2802multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2803 MUBUF_Real_AllAddr_vi<op, 0> { 2804 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2805 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2806 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2807 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2808} 2809 2810defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2811defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2812defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2813defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2814defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2815defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2816defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2817defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2818let SubtargetPredicate = HasUnpackedD16VMem in { 2819 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2820 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2821 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2822 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2823 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2824 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2825 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2826 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2827} // End HasUnpackedD16VMem. 2828let SubtargetPredicate = HasPackedD16VMem in { 2829 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2830 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2831 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2832 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2833 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2834 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2835 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2836 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2837} // End HasPackedD16VMem. 2838defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2839defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2840defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2841defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2842defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2843defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 2844defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 2845defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 2846defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2847defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2848defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2849defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2850defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2851defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2852defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2853defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2854 2855defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2856defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2857defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2858defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2859defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2860defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2861 2862defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2863defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2864 2865defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2866defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2867defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2868defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2869defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2870defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2871defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2872defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2873defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2874defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2875defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2876defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2877defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2878 2879defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2880defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2881defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2882defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2883defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2884defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2885defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2886defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2887defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2888defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2889defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2890defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2891defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2892 2893defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2894 2895let AssemblerPredicate = isGFX8GFX9 in { 2896def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2897def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2898} // End AssemblerPredicate = isGFX8GFX9 2899 2900let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 2901 2902defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2903defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2904 2905} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 2906 2907let SubtargetPredicate = isGFX90APlus in { 2908 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2909 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2910 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2911} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2912 2913def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2914 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2915 let AssemblerPredicate = isGFX90AOnly; 2916 let SubtargetPredicate = isGFX90AOnly; 2917} 2918def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2919 2920let SubtargetPredicate = isGFX940Plus in { 2921def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2922def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2923} 2924 2925class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2926 MTBUF_Real<ps>, 2927 Enc64, 2928 SIMCInstr<ps.PseudoInstr, Enc> { 2929 2930 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2931 let Inst{12} = ps.offen; 2932 let Inst{13} = ps.idxen; 2933 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2934 let Inst{18-15} = op; 2935 let Inst{22-19} = dfmt; 2936 let Inst{25-23} = nfmt; 2937 let Inst{31-26} = 0x3a; //encoding 2938 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2939 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2940 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2941 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2942 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2943 let Inst{55} = ps.tfe; 2944 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2945} 2946 2947class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2948 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2949 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2950 let DecoderNamespace = "GFX8"; 2951 2952 let Inst{55} = ps.tfe; 2953} 2954 2955class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2956 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2957 let AssemblerPredicate = isGFX90APlus; 2958 let DecoderNamespace = "GFX90A"; 2959 let AsmString = ps.Mnemonic # ps.AsmOperands; 2960 2961 let Inst{55} = acc; 2962} 2963 2964multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2965 def _vi : MTBUF_Real_vi<op, ps>; 2966 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2967} 2968 2969multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2970 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2971 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2972 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2973 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2974} 2975 2976class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2977 MTBUF_Real<ps>, 2978 Enc64, 2979 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2980 let AssemblerPredicate=HasUnpackedD16VMem; 2981 let DecoderNamespace="GFX80_UNPACKED"; 2982 2983 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2984 let Inst{12} = ps.offen; 2985 let Inst{13} = ps.idxen; 2986 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2987 let Inst{18-15} = op; 2988 let Inst{22-19} = dfmt; 2989 let Inst{25-23} = nfmt; 2990 let Inst{31-26} = 0x3a; //encoding 2991 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2992 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2993 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2994 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2995 let Inst{55} = ps.tfe; 2996 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2997} 2998 2999multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 3000 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 3001 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 3002 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 3003 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 3004} 3005 3006defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 3007defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 3008defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 3009defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 3010defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 3011defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 3012defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 3013defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 3014let SubtargetPredicate = HasUnpackedD16VMem in { 3015 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 3016 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 3017 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 3018 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 3019 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 3020 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 3021 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 3022 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 3023} // End HasUnpackedD16VMem. 3024let SubtargetPredicate = HasPackedD16VMem in { 3025 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 3026 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 3027 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 3028 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 3029 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 3030 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 3031 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 3032 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 3033} // End HasUnpackedD16VMem. 3034 3035def MUBUFInfoTable : GenericTable { 3036 let FilterClass = "MUBUF_Pseudo"; 3037 let CppTypeName = "MUBUFInfo"; 3038 let Fields = [ 3039 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3040 "IsBufferInv" 3041 ]; 3042 3043 let PrimaryKey = ["Opcode"]; 3044 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3045} 3046 3047def getMUBUFInfoFromOpcode : SearchIndex { 3048 let Table = MUBUFInfoTable; 3049 let Key = ["Opcode"]; 3050} 3051 3052def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3053 let Table = MUBUFInfoTable; 3054 let Key = ["BaseOpcode", "elements"]; 3055} 3056 3057def MTBUFInfoTable : GenericTable { 3058 let FilterClass = "MTBUF_Pseudo"; 3059 let CppTypeName = "MTBUFInfo"; 3060 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3061 3062 let PrimaryKey = ["Opcode"]; 3063 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3064} 3065 3066def getMTBUFInfoFromOpcode : SearchIndex { 3067 let Table = MTBUFInfoTable; 3068 let Key = ["Opcode"]; 3069} 3070 3071def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3072 let Table = MTBUFInfoTable; 3073 let Key = ["BaseOpcode", "elements"]; 3074} 3075