1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MTBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43 44//===----------------------------------------------------------------------===// 45// BUF class (base class for MTBUF and MUBUF pseudos) 46//===----------------------------------------------------------------------===// 47 48class BUF_Pseudo <string opName, dag outs, dag ins, 49 string asmOps, list<dag> pattern=[]> : 50 InstSI<outs, ins, "", pattern>, 51 SIMCInstr<opName, SIEncodingFamily.NONE> { 52 53 let isPseudo = 1; 54 let isCodeGenOnly = 1; 55 let Size = 8; 56 let UseNamedOperandTable = 1; 57 58 string Mnemonic = opName; 59 string AsmOperands = asmOps; 60 61 Instruction Opcode = !cast<Instruction>(NAME); 62 63 64 let VM_CNT = 1; 65 let EXP_CNT = 1; 66 67 let Uses = [EXEC]; 68 let hasSideEffects = 0; 69 let SchedRW = [WriteVMEM]; 70 71 72 73 bits<1> offen = 0; 74 bits<1> idxen = 0; 75 bits<1> addr64 = 0; 76 bits<1> lds = 0; 77 bits<1> has_vdata = !not(lds); 78 bits<1> has_vaddr = 1; 79 bits<1> has_glc = 1; 80 bits<1> has_dlc = 1; 81 bits<1> glc_value = 0; // the value for glc if no such operand 82 bits<1> dlc_value = 0; // the value for dlc if no such operand 83 bits<1> has_srsrc = 1; 84 bits<1> has_soffset = 1; 85 bits<1> has_offset = 1; 86 bits<1> has_slc = 1; 87 bits<1> tfe = ?; 88 bits<4> elements = 0; 89 bits<1> has_sccb = 1; 90 bits<1> sccb_value = 0; 91 bits<1> IsBufferInv = 0; 92} 93 94 95 96//===----------------------------------------------------------------------===// 97// MTBUF classes 98//===----------------------------------------------------------------------===// 99 100class MTBUFGetBaseOpcode<string Op> { 101 string ret = !subst("FORMAT_XY", "FORMAT_X", 102 !subst("FORMAT_XYZ", "FORMAT_X", 103 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 104} 105 106 107class MTBUF_Pseudo <string opName, dag outs, dag ins, 108 string asmOps, list<dag> pattern=[]> : 109 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 110 111 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 112 let MTBUF = 1; 113} 114 115class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 116 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 117 118 let isPseudo = 0; 119 let isCodeGenOnly = 0; 120 121 let VM_CNT = 1; 122 let EXP_CNT = 1; 123 let MTBUF = 1; 124 125 // copy relevant pseudo op flags 126 let UseNamedOperandTable = ps.UseNamedOperandTable; 127 let SubtargetPredicate = ps.SubtargetPredicate; 128 let AsmMatchConverter = ps.AsmMatchConverter; 129 let Constraints = ps.Constraints; 130 let DisableEncoding = ps.DisableEncoding; 131 let TSFlags = ps.TSFlags; 132 let SchedRW = ps.SchedRW; 133 let mayLoad = ps.mayLoad; 134 let mayStore = ps.mayStore; 135 let IsAtomicRet = ps.IsAtomicRet; 136 let IsAtomicNoRet = ps.IsAtomicNoRet; 137 138 bits<12> offset; 139 bits<5> cpol; 140 bits<7> format; 141 bits<8> vaddr; 142 bits<10> vdata; 143 bits<7> srsrc; 144 bits<8> soffset; 145 146 bits<4> dfmt = format{3-0}; 147 bits<3> nfmt = format{6-4}; 148 149 // GFX90A+ only: instruction uses AccVGPR for data 150 // Bit supersedes tfe. 151 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 152} 153 154class getMTBUFInsDA<list<RegisterClass> vdataList, 155 list<RegisterClass> vaddrList=[]> { 156 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 157 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 158 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 159 160 dag NonVaddrInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, i1imm:$swz); 161 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 162 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 163} 164 165 166class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 167 dag ret = 168 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 169 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 170 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 171 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 172 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 173 (ins)))))); 174} 175 176class getMTBUFAsmOps<int addrKind> { 177 string Pfx = 178 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 179 !if(!eq(addrKind, BUFAddrKind.OffEn), 180 "$vaddr, $srsrc,$format $soffset offen", 181 !if(!eq(addrKind, BUFAddrKind.IdxEn), 182 "$vaddr, $srsrc,$format $soffset idxen", 183 !if(!eq(addrKind, BUFAddrKind.BothEn), 184 "$vaddr, $srsrc,$format $soffset idxen offen", 185 !if(!eq(addrKind, BUFAddrKind.Addr64), 186 "$vaddr, $srsrc,$format $soffset addr64", 187 ""))))); 188 string ret = " $vdata, " # Pfx # "$offset$cpol"; 189} 190 191class MTBUF_SetupAddr<int addrKind> { 192 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 193 !eq(addrKind, BUFAddrKind.BothEn)); 194 195 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 196 !eq(addrKind, BUFAddrKind.BothEn)); 197 198 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 199 200 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 201} 202 203class MTBUF_Load_Pseudo <string opName, 204 int addrKind, 205 RegisterClass vdataClass, 206 int elems, 207 list<dag> pattern=[], 208 // Workaround bug bz30254 209 int addrKindCopy = addrKind> 210 : MTBUF_Pseudo<opName, 211 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 212 getMTBUFIns<addrKindCopy>.ret, 213 getMTBUFAsmOps<addrKindCopy>.ret, 214 pattern>, 215 MTBUF_SetupAddr<addrKindCopy> { 216 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 217 let mayLoad = 1; 218 let mayStore = 0; 219 let elements = elems; 220} 221 222multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 223 int elems> { 224 225 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 226 MTBUFAddr64Table<0, NAME>; 227 228 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 229 MTBUFAddr64Table<1, NAME>; 230 231 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 235 let DisableWQM = 1 in { 236 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 237 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 238 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 239 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 240 } 241} 242 243class MTBUF_Store_Pseudo <string opName, 244 int addrKind, 245 RegisterClass vdataClass, 246 int elems, 247 list<dag> pattern=[], 248 // Workaround bug bz30254 249 int addrKindCopy = addrKind, 250 RegisterClass vdataClassCopy = vdataClass> 251 : MTBUF_Pseudo<opName, 252 (outs), 253 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 254 getMTBUFAsmOps<addrKindCopy>.ret, 255 pattern>, 256 MTBUF_SetupAddr<addrKindCopy> { 257 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 258 let mayLoad = 0; 259 let mayStore = 1; 260 let elements = elems; 261} 262 263multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 264 int elems> { 265 266 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 267 MTBUFAddr64Table<0, NAME>; 268 269 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 270 MTBUFAddr64Table<1, NAME>; 271 272 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 273 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 274 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 275 276 let DisableWQM = 1 in { 277 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 278 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 279 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 280 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 281 } 282} 283 284 285//===----------------------------------------------------------------------===// 286// MUBUF classes 287//===----------------------------------------------------------------------===// 288 289class MUBUFGetBaseOpcode<string Op> { 290 string ret = !subst("DWORDX2", "DWORD", 291 !subst("DWORDX3", "DWORD", 292 !subst("DWORDX4", "DWORD", Op))); 293} 294 295class MUBUF_Pseudo <string opName, dag outs, dag ins, 296 string asmOps, list<dag> pattern=[]> : 297 BUF_Pseudo <opName, outs, ins, asmOps, pattern> { 298 299 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 300 let MUBUF = 1; 301 let AsmMatchConverter = "cvtMubuf"; 302} 303 304class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 305 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 306 307 let isPseudo = 0; 308 let isCodeGenOnly = 0; 309 310 let VM_CNT = 1; 311 let EXP_CNT = 1; 312 let MUBUF = 1; 313 314 // copy relevant pseudo op flags 315 let SubtargetPredicate = ps.SubtargetPredicate; 316 let AsmMatchConverter = ps.AsmMatchConverter; 317 let OtherPredicates = ps.OtherPredicates; 318 let Constraints = ps.Constraints; 319 let DisableEncoding = ps.DisableEncoding; 320 let TSFlags = ps.TSFlags; 321 let UseNamedOperandTable = ps.UseNamedOperandTable; 322 let SchedRW = ps.SchedRW; 323 let mayLoad = ps.mayLoad; 324 let mayStore = ps.mayStore; 325 let IsAtomicRet = ps.IsAtomicRet; 326 let IsAtomicNoRet = ps.IsAtomicNoRet; 327 let VALU = ps.VALU; 328 let LGKM_CNT = ps.LGKM_CNT; 329 330 bits<12> offset; 331 bits<5> cpol; 332 bits<8> vaddr; 333 bits<10> vdata; 334 bits<7> srsrc; 335 bits<8> soffset; 336 337 // GFX90A+ only: instruction uses AccVGPR for data 338 // Bit supersedes tfe. 339 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 340} 341 342// For cache invalidation instructions. 343class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 344 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 345 346 let AsmMatchConverter = ""; 347 348 let hasSideEffects = 1; 349 let mayLoad = 0; 350 let mayStore = 0; 351 352 let IsBufferInv = 1; 353 // Set everything else to 0. 354 let offen = 0; 355 let idxen = 0; 356 let addr64 = 0; 357 let has_vdata = 0; 358 let has_vaddr = 0; 359 let has_glc = 0; 360 let has_dlc = 0; 361 let glc_value = 0; 362 let dlc_value = 0; 363 let has_srsrc = 0; 364 let has_soffset = 0; 365 let has_offset = 0; 366 let has_slc = 0; 367 let has_sccb = 0; 368 let sccb_value = 0; 369} 370 371class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> { 372 RegisterOperand tfeVDataOp = 373 !if(!eq(RC.Size, 32), AVLdSt_64, 374 !if(!eq(RC.Size, 64), AVLdSt_96, 375 !if(!eq(RC.Size, 96), AVLdSt_128, 376 !if(!eq(RC.Size, 128), AVLdSt_160, 377 RegisterOperand<VReg_1> // Invalid register. 378 )))); 379 380 RegisterOperand ret = !if(isTFE, tfeVDataOp, getLdStRegisterOperand<RC>.ret); 381} 382 383class getMUBUFInsDA<list<RegisterClass> vdataList, 384 list<RegisterClass> vaddrList, bit isTFE> { 385 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 386 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 387 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret; 388 389 dag NonVaddrInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol, i1imm_0:$swz); 390 dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs)); 391 dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs)); 392} 393 394class getMUBUFElements<ValueType vt> { 395 int ret = 396 !if(!eq(vt, f16), 1, 397 !if(!eq(vt, v2f16), 2, 398 !if(!eq(vt, v3f16), 3, 399 !if(!eq(vt, v4f16), 4, 400 !if(!eq(vt.Size, 32), 1, 401 !if(!eq(vt.Size, 64), 2, 402 !if(!eq(vt.Size, 96), 3, 403 !if(!eq(vt.Size, 128), 4, 0) 404 ) 405 ) 406 ) 407 ) 408 ) 409 ) 410 ); 411} 412 413class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE> { 414 dag ret = 415 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE>.ret, 416 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE>.ret, 417 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isTFE>.ret, 418 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE>.ret, 419 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE>.ret, 420 (ins)))))); 421} 422 423class getMUBUFAsmOps<int addrKind, bit noVdata = 0, bit isLds = 0, bit isTFE = 0> { 424 string Vdata = !if(noVdata, " ", " $vdata, "); 425 string Lds = !if(isLds, " lds", ""); 426 string TFE = !if(isTFE, " tfe", ""); 427 string MainArgs = 428 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 429 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 430 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 431 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 432 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 433 ""))))); 434 string Offset = "$offset"; 435 string OtherArgs = "$cpol"; 436 437 string ret = Vdata # MainArgs # Offset # OtherArgs # Lds # TFE; 438} 439 440class MUBUF_SetupAddr<int addrKind> { 441 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 442 !eq(addrKind, BUFAddrKind.BothEn)); 443 444 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 445 !eq(addrKind, BUFAddrKind.BothEn)); 446 447 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 448 449 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 450} 451 452class MUBUF_Load_Pseudo <string opName, 453 int addrKind, 454 ValueType vdata_vt, 455 bit HasTiedDest = 0, 456 bit isLds = 0, 457 bit isLdsOpc = 0, 458 bit isTFE = 0, 459 list<dag> pattern=[], 460 // Workaround bug bz30254 461 int addrKindCopy = addrKind, 462 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 463 RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret> 464 : MUBUF_Pseudo<opName, 465 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 466 !con(getMUBUFIns<addrKindCopy, [], isTFE>.ret, 467 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 468 getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret, 469 pattern>, 470 MUBUF_SetupAddr<addrKindCopy> { 471 let PseudoInstr = opName # !if(isLds, "_lds", "") # !if(isTFE, "_tfe", "") # 472 "_" # getAddrName<addrKindCopy>.ret; 473 let AsmMatchConverter = "cvtMubuf"; 474 475 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 476 let LGKM_CNT = isLds; 477 let has_vdata = !not(!or(isLds, isLdsOpc)); 478 let mayLoad = 1; 479 let mayStore = isLds; 480 let maybeAtomic = 1; 481 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 482 let tfe = isTFE; 483 let lds = isLds; 484 let elements = getMUBUFElements<vdata_vt>.ret; 485 let VALU = isLds; 486} 487 488class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 489 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 490 (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset)) 491>; 492 493class MUBUF_Addr64_Load_Pat <Instruction inst, 494 ValueType load_vt = i32, 495 SDPatternOperator ld = null_frag> : Pat < 496 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 497 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)) 498>; 499 500multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 501 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 502 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 503} 504 505multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt, 506 bit TiedDest, bit isLds, bit isTFE> { 507 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 508 509 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE>, 510 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 511 512 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE>, 513 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 514 515 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 516 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 517 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 518 519 let DisableWQM = 1 in { 520 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE>; 521 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 522 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 523 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE>; 524 } 525} 526 527multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32, 528 bit TiedDest = 0, bit isLds = 0> { 529 defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0>; 530 if !not(isLds) then 531 defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1>; 532} 533 534multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 535 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 536 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 537} 538 539multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 540 ValueType load_vt = i32, 541 bit TiedDest = 0, 542 bit isLds = 0, 543 bit isLdsOpc = 1> { 544 545 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 546 547 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 548 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 549 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 550 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 551} 552 553class MUBUF_Store_Pseudo <string opName, 554 int addrKind, 555 ValueType store_vt, 556 bit isTFE = 0, 557 list<dag> pattern=[], 558 // Workaround bug bz30254 559 int addrKindCopy = addrKind> 560 : MUBUF_Pseudo<opName, 561 (outs), 562 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret], isTFE>.ret, 563 getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret, 564 pattern>, 565 MUBUF_SetupAddr<addrKindCopy> { 566 let PseudoInstr = opName # "_" # !if(isTFE, "_tfe", "") # 567 getAddrName<addrKindCopy>.ret; 568 let mayLoad = 0; 569 let mayStore = 1; 570 let maybeAtomic = 1; 571 let elements = getMUBUFElements<store_vt>.ret; 572 let tfe = isTFE; 573} 574 575multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt, 576 SDPatternOperator st, bit isTFE> { 577 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 578 579 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, 580 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 581 i32:$offset))]>, 582 MUBUFAddr64Table<0, NAME>; 583 584 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, 585 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 586 i32:$offset))]>, 587 MUBUFAddr64Table<1, NAME>; 588 589 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE>; 590 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE>; 591 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE>; 592 593 let DisableWQM = 1 in { 594 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE>; 595 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE>; 596 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE>; 597 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE>; 598 } 599} 600 601multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32, 602 SDPatternOperator st = null_frag> { 603 defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, st, 0>; 604 defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, null_frag, 1>; 605} 606 607class MUBUF_Pseudo_Store_Lds<string opName> 608 : MUBUF_Pseudo<opName, 609 (outs), 610 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, i1imm:$swz), 611 " $srsrc, $soffset$offset lds$cpol"> { 612 let LGKM_CNT = 1; 613 let mayLoad = 1; 614 let mayStore = 1; 615 let maybeAtomic = 1; 616 617 let has_vdata = 0; 618 let has_vaddr = 0; 619 let lds = 1; 620 let VALU = 1; 621 622 let Uses = [EXEC, M0]; 623 let AsmMatchConverter = "cvtMubuf"; 624} 625 626class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 627 list<RegisterClass> vaddrList=[]> { 628 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 629 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 630 631 dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata)); 632 dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr))); 633 dag MainInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset); 634 dag CPol = !if(vdata_in, (ins CPol_GLC1:$cpol), (ins CPol_0:$cpol)); 635 636 dag ret = !con(Data, MainInputs, CPol); 637} 638 639class getMUBUFAtomicIns<int addrKind, 640 RegisterClass vdataClass, 641 bit vdata_in, 642 // Workaround bug bz30254 643 RegisterClass vdataClassCopy=vdataClass> { 644 dag ret = 645 !if(!eq(addrKind, BUFAddrKind.Offset), 646 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 647 !if(!eq(addrKind, BUFAddrKind.OffEn), 648 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 649 !if(!eq(addrKind, BUFAddrKind.IdxEn), 650 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 651 !if(!eq(addrKind, BUFAddrKind.BothEn), 652 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 653 !if(!eq(addrKind, BUFAddrKind.Addr64), 654 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 655 (ins)))))); 656} 657 658class MUBUF_Atomic_Pseudo<string opName, 659 int addrKind, 660 dag outs, 661 dag ins, 662 string asmOps, 663 list<dag> pattern=[], 664 // Workaround bug bz30254 665 int addrKindCopy = addrKind> 666 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 667 MUBUF_SetupAddr<addrKindCopy> { 668 let mayStore = 1; 669 let mayLoad = 1; 670 let hasPostISelHook = 1; 671 let hasSideEffects = 1; 672 let DisableWQM = 1; 673 let has_glc = 0; 674 let has_dlc = 0; 675 let has_sccb = 1; 676 let maybeAtomic = 1; 677 let AsmMatchConverter = "cvtMubufAtomic"; 678} 679 680class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 681 RegisterClass vdataClass, 682 list<dag> pattern=[], 683 // Workaround bug bz30254 684 int addrKindCopy = addrKind, 685 RegisterClass vdataClassCopy = vdataClass> 686 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 687 (outs), 688 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 689 getMUBUFAsmOps<addrKindCopy>.ret, 690 pattern>, 691 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 692 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 693 let glc_value = 0; 694 let dlc_value = 0; 695 let sccb_value = 0; 696 let IsAtomicNoRet = 1; 697} 698 699class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 700 RegisterClass vdataClass, 701 list<dag> pattern=[], 702 // Workaround bug bz30254 703 int addrKindCopy = addrKind, 704 RegisterClass vdataClassCopy = vdataClass, 705 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 706 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 707 (outs vdata_op:$vdata), 708 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 709 getMUBUFAsmOps<addrKindCopy>.ret, 710 pattern>, 711 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 712 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 713 let glc_value = 1; 714 let dlc_value = 0; 715 let sccb_value = 0; 716 let IsAtomicRet = 1; 717 let Constraints = "$vdata = $vdata_in"; 718 let DisableEncoding = "$vdata_in"; 719} 720 721multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 722 RegisterClass vdataClass, 723 ValueType vdataType, 724 bit isFP = isFloatType<vdataType>.ret> { 725 let FPAtomic = isFP in { 726 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 727 MUBUFAddr64Table <0, NAME>; 728 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 729 MUBUFAddr64Table <1, NAME>; 730 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 731 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 732 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 733 } 734} 735 736multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 737 RegisterClass vdataClass, 738 ValueType vdataType, 739 SDPatternOperator atomic, 740 bit isFP = isFloatType<vdataType>.ret> { 741 let FPAtomic = isFP in { 742 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 743 [(set vdataType:$vdata, 744 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), 745 vdataType:$vdata_in))]>, 746 MUBUFAddr64Table <0, NAME # "_RTN">; 747 748 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 749 [(set vdataType:$vdata, 750 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 751 vdataType:$vdata_in))]>, 752 MUBUFAddr64Table <1, NAME # "_RTN">; 753 754 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 755 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 756 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 757 } 758} 759 760multiclass MUBUF_Pseudo_Atomics <string opName, 761 RegisterClass vdataClass, 762 ValueType vdataType, 763 SDPatternOperator atomic = null_frag> : 764 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 765 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 766 767 768//===----------------------------------------------------------------------===// 769// MUBUF Instructions 770//===----------------------------------------------------------------------===// 771 772defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 773 "buffer_load_format_x", f32 774>; 775defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 776 "buffer_load_format_xy", v2f32 777>; 778defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 779 "buffer_load_format_xyz", v3f32 780>; 781defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 782 "buffer_load_format_xyzw", v4f32 783>; 784defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 785 "buffer_store_format_x", f32 786>; 787defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 788 "buffer_store_format_xy", v2f32 789>; 790defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 791 "buffer_store_format_xyz", v3f32 792>; 793defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 794 "buffer_store_format_xyzw", v4f32 795>; 796 797let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 798let TiedSourceNotRead = 1 in { 799 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 800 "buffer_load_format_d16_x", i32 801 >; 802 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 803 "buffer_load_format_d16_xy", v2i32 804 >; 805 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 806 "buffer_load_format_d16_xyz", v3i32 807 >; 808 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 809 "buffer_load_format_d16_xyzw", v4i32 810 >; 811} 812 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 813 "buffer_store_format_d16_x", i32 814 >; 815 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 816 "buffer_store_format_d16_xy", v2i32 817 >; 818 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 819 "buffer_store_format_d16_xyz", v3i32 820 >; 821 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 822 "buffer_store_format_d16_xyzw", v4i32 823 >; 824} // End HasUnpackedD16VMem. 825 826let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 827let TiedSourceNotRead = 1 in { 828 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 829 "buffer_load_format_d16_x", f16 830 >; 831 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 832 "buffer_load_format_d16_xy", v2f16 833 >; 834 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 835 "buffer_load_format_d16_xyz", v3f16 836 >; 837 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 838 "buffer_load_format_d16_xyzw", v4f16 839 >; 840} 841 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 842 "buffer_store_format_d16_x", f16 843 >; 844 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 845 "buffer_store_format_d16_xy", v2f16 846 >; 847 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 848 "buffer_store_format_d16_xyz", v3f16 849 >; 850 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 851 "buffer_store_format_d16_xyzw", v4f16 852 >; 853} // End HasPackedD16VMem. 854 855defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 856 "buffer_load_ubyte", i32 857>; 858defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 859 "buffer_load_sbyte", i32 860>; 861defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 862 "buffer_load_ushort", i32 863>; 864defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 865 "buffer_load_sshort", i32 866>; 867defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 868 "buffer_load_dword", i32 869>; 870defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 871 "buffer_load_dwordx2", v2i32 872>; 873defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 874 "buffer_load_dwordx3", v3i32 875>; 876defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 877 "buffer_load_dwordx4", v4i32 878>; 879 880defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 881 "buffer_load_lds_b32", i32 882>; 883defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 884 "buffer_load_lds_format_x", f32 885>; 886defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 887 "buffer_load_lds_i8", i32 888>; 889defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 890 "buffer_load_lds_i16", i32 891>; 892defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 893 "buffer_load_lds_u8", i32 894>; 895defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 896 "buffer_load_lds_u16", i32 897>; 898 899defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 900defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 901defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 902defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 903defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 904defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 905defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 906defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 907defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 908defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 909defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 910defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 911defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 912defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 913 914defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 915 "buffer_store_byte", i32, truncstorei8_global 916>; 917defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 918 "buffer_store_short", i32, truncstorei16_global 919>; 920defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 921 "buffer_store_dword", i32, store_global 922>; 923defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 924 "buffer_store_dwordx2", v2i32, store_global 925>; 926defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 927 "buffer_store_dwordx3", v3i32, store_global 928>; 929defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 930 "buffer_store_dwordx4", v4i32, store_global 931>; 932defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 933 "buffer_atomic_swap", VGPR_32, i32 934>; 935defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 936 "buffer_atomic_cmpswap", VReg_64, v2i32 937>; 938defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 939 "buffer_atomic_add", VGPR_32, i32 940>; 941defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 942 "buffer_atomic_sub", VGPR_32, i32 943>; 944defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 945 "buffer_atomic_smin", VGPR_32, i32 946>; 947defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 948 "buffer_atomic_umin", VGPR_32, i32 949>; 950defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 951 "buffer_atomic_smax", VGPR_32, i32 952>; 953defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 954 "buffer_atomic_umax", VGPR_32, i32 955>; 956defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 957 "buffer_atomic_and", VGPR_32, i32 958>; 959defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 960 "buffer_atomic_or", VGPR_32, i32 961>; 962defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 963 "buffer_atomic_xor", VGPR_32, i32 964>; 965defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 966 "buffer_atomic_inc", VGPR_32, i32 967>; 968defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 969 "buffer_atomic_dec", VGPR_32, i32 970>; 971defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 972 "buffer_atomic_swap_x2", VReg_64, i64 973>; 974defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 975 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 976>; 977defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 978 "buffer_atomic_add_x2", VReg_64, i64 979>; 980defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 981 "buffer_atomic_sub_x2", VReg_64, i64 982>; 983defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 984 "buffer_atomic_smin_x2", VReg_64, i64 985>; 986defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 987 "buffer_atomic_umin_x2", VReg_64, i64 988>; 989defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 990 "buffer_atomic_smax_x2", VReg_64, i64 991>; 992defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 993 "buffer_atomic_umax_x2", VReg_64, i64 994>; 995defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 996 "buffer_atomic_and_x2", VReg_64, i64 997>; 998defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 999 "buffer_atomic_or_x2", VReg_64, i64 1000>; 1001defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1002 "buffer_atomic_xor_x2", VReg_64, i64 1003>; 1004defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1005 "buffer_atomic_inc_x2", VReg_64, i64 1006>; 1007defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1008 "buffer_atomic_dec_x2", VReg_64, i64 1009>; 1010 1011let SubtargetPredicate = HasGFX10_BEncoding in 1012defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1013 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1014>; 1015 1016let SubtargetPredicate = isGFX8GFX9 in { 1017def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1018} 1019 1020let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1021/* 1022defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1023defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1024*/ 1025 1026def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1027 int_amdgcn_buffer_wbinvl1_sc>; 1028} 1029 1030let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1031 1032defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1033 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1034>; 1035defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1036 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1037>; 1038defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1039 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1040>; 1041 1042} 1043 1044let SubtargetPredicate = isGFX6GFX7GFX10 in { 1045 1046defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1047 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1048>; 1049defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1050 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1051>; 1052defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1053 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1054>; 1055 1056} 1057 1058let SubtargetPredicate = HasD16LoadStore in { 1059let TiedSourceNotRead = 1 in { 1060 1061defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1062 "buffer_load_ubyte_d16", i32, 1 1063>; 1064 1065defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1066 "buffer_load_ubyte_d16_hi", i32, 1 1067>; 1068 1069defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1070 "buffer_load_sbyte_d16", i32, 1 1071>; 1072 1073defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1074 "buffer_load_sbyte_d16_hi", i32, 1 1075>; 1076 1077defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1078 "buffer_load_short_d16", i32, 1 1079>; 1080 1081defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1082 "buffer_load_short_d16_hi", i32, 1 1083>; 1084 1085defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1086 "buffer_load_format_d16_hi_x", i32 1087>; 1088} // End TiedSourceNotRead 1089 1090defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1091 "buffer_store_byte_d16_hi", i32 1092>; 1093 1094defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1095 "buffer_store_short_d16_hi", i32 1096>; 1097 1098defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1099 "buffer_store_format_d16_hi_x", i32 1100>; 1101 1102} // End HasD16LoadStore 1103 1104def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1105 int_amdgcn_buffer_wbinvl1>; 1106 1107let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1108defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1109 "buffer_atomic_add_f32", VGPR_32, f32 1110>; 1111 1112let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in 1113defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1114 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1115>; 1116 1117let OtherPredicates = [HasAtomicFaddRtnInsts] in 1118defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1119 "buffer_atomic_add_f32", VGPR_32, f32, null_frag 1120>; 1121 1122let OtherPredicates = [HasAtomicBufferGlobalPkAddF16Insts] in 1123defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1124 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, null_frag 1125>; 1126 1127//===----------------------------------------------------------------------===// 1128// MTBUF Instructions 1129//===----------------------------------------------------------------------===// 1130 1131defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1132defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1133defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1134defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1135defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1136defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1137defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1138defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1139 1140let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1141let TiedSourceNotRead = 1 in { 1142 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1143 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1144 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1145 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1146} 1147 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1148 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1149 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1150 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1151} // End HasUnpackedD16VMem. 1152 1153let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1154let TiedSourceNotRead = 1 in { 1155 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1156 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1157 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1158 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1159} 1160 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1161 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1162 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1163 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1164} // End HasPackedD16VMem. 1165 1166let SubtargetPredicate = isGFX7Plus in { 1167 1168//===----------------------------------------------------------------------===// 1169// Instruction definitions for CI and newer. 1170//===----------------------------------------------------------------------===// 1171 1172def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1173 int_amdgcn_buffer_wbinvl1_vol>; 1174 1175} // End let SubtargetPredicate = isGFX7Plus 1176 1177let SubtargetPredicate = isGFX90APlus in { 1178 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1179 let has_glc = 1; 1180 let has_sccb = 1; 1181 let InOperandList = (ins CPol_0:$cpol); 1182 let AsmOperands = "$cpol"; 1183 } 1184 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1185 let SubtargetPredicate = isGFX90AOnly; 1186 } 1187 1188 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1189 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1190 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1191} // End SubtargetPredicate = isGFX90APlus 1192 1193def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1194 let SubtargetPredicate = isGFX940Plus; 1195 let has_glc = 1; 1196 let has_sccb = 1; 1197 let InOperandList = (ins CPol_0:$cpol); 1198 let AsmOperands = "$cpol"; 1199} 1200 1201let SubtargetPredicate = isGFX10Plus in { 1202 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1203 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1204} // End SubtargetPredicate = isGFX10Plus 1205 1206//===----------------------------------------------------------------------===// 1207// MUBUF Patterns 1208//===----------------------------------------------------------------------===// 1209 1210//===----------------------------------------------------------------------===// 1211// buffer_load/store_format patterns 1212//===----------------------------------------------------------------------===// 1213 1214multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1215 string opcode, ValueType memoryVt = vt> { 1216 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1217 1218 def : GCNPat< 1219 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1220 timm:$auxiliary, 0)), 1221 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1222 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1223 >; 1224 1225 def : GCNPat< 1226 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1227 timm:$auxiliary, 0)), 1228 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1229 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1230 >; 1231 1232 def : GCNPat< 1233 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1234 timm:$auxiliary, timm)), 1235 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1236 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1237 >; 1238 1239 def : GCNPat< 1240 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1241 timm:$auxiliary, timm)), 1242 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1243 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1244 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1245 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1246 >; 1247} 1248 1249defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1250defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1251defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1252defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1253defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1254defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1255defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1256defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1257 1258defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v2i32, "BUFFER_LOAD_FORMAT_X_TFE">; 1259defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v3i32, "BUFFER_LOAD_FORMAT_XY_TFE">; 1260defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v4i32, "BUFFER_LOAD_FORMAT_XYZ_TFE">; 1261defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_tfe, v5i32, "BUFFER_LOAD_FORMAT_XYZW_TFE">; 1262 1263let SubtargetPredicate = HasUnpackedD16VMem in { 1264 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1265 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1266 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1267 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1268 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1269 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1270} // End HasUnpackedD16VMem. 1271 1272let SubtargetPredicate = HasPackedD16VMem in { 1273 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1274 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1275 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1276 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1277 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1278 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1279 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1280 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1281 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1282} // End HasPackedD16VMem. 1283 1284defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1285defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1288defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1289defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1290defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1291defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1292defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1293defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1294defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1295defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1296defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1297defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1298defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1299defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1300 1301multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1302 string opcode, ValueType memoryVt = vt> { 1303 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1304 1305 def : GCNPat< 1306 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1307 timm:$auxiliary, 0), 1308 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1309 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1310 >; 1311 1312 def : GCNPat< 1313 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1314 timm:$auxiliary, 0), 1315 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1316 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1317 >; 1318 1319 def : GCNPat< 1320 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1321 timm:$auxiliary, timm), 1322 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1323 timm:$offset, (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1324 >; 1325 1326 def : GCNPat< 1327 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1328 timm:$auxiliary, timm), 1329 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1330 getVregSrcForVT<vt>.ret:$vdata, 1331 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1332 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (extract_cpol $auxiliary), 1333 (extract_swz $auxiliary)) 1334 >; 1335} 1336 1337defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1338defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1339defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1340defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1341defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1342defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1343defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1344defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1345 1346let SubtargetPredicate = HasUnpackedD16VMem in { 1347 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1348 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1349 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1350 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1351 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1352 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1353} // End HasUnpackedD16VMem. 1354 1355let SubtargetPredicate = HasPackedD16VMem in { 1356 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1357 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1358 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1359 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1360 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1361 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1362 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1363 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1364 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1365} // End HasPackedD16VMem. 1366 1367defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1368defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1369defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1370defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1371defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1372defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1373defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1374defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1375defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1376defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1377defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1378defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1379defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1380defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1381 1382//===----------------------------------------------------------------------===// 1383// buffer_atomic patterns 1384//===----------------------------------------------------------------------===// 1385 1386multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1387 foreach RtnMode = ["ret", "noret"] in { 1388 1389 defvar Op = !cast<SDPatternOperator>(OpPrefix 1390 # !if(!eq(RtnMode, "ret"), "", "_noret") 1391 # !if(isIntr, "", "_" # vt.Size)); 1392 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1393 1394 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1395 def : GCNPat< 1396 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)), 1397 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1398 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1399 >; 1400 1401 def : GCNPat< 1402 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1403 vt:$vdata_in)), 1404 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1405 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1406 >; 1407 } // end let AddedComplexity 1408 1409 } // end foreach RtnMode 1410} 1411 1412multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1413 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1414} 1415 1416multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1417 foreach RtnMode = ["ret", "noret"] in { 1418 1419 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1420 # !if(!eq(RtnMode, "ret"), "", "_noret") 1421 # "_" # vt.Size); 1422 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1423 1424 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1425 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1426 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1427 offset:$offset); 1428 def : GCNPat< 1429 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)), 1430 !if(!eq(RtnMode, "ret"), 1431 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1432 !if(!eq(vt, i32), sub0, sub0_sub1)), 1433 OffsetResDag) 1434 >; 1435 1436 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1437 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1438 SCSrc_b32:$soffset, offset:$offset); 1439 def : GCNPat< 1440 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), 1441 data_vt:$vdata_in)), 1442 !if(!eq(RtnMode, "ret"), 1443 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1444 !if(!eq(vt, i32), sub0, sub0_sub1)), 1445 Addr64ResDag) 1446 >; 1447 } // end let AddedComplexity 1448 1449 } // end foreach RtnMode 1450} 1451 1452foreach Ty = [i32, i64] in { 1453 1454defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1455 1456defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1457defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1458defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1459defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1460defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1461defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1462defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1463defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1464defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1465defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1466defm : BufferAtomicPat<"atomic_load_uinc_wrap_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1467defm : BufferAtomicPat<"atomic_load_udec_wrap_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1468 1469} // end foreach Ty 1470 1471defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1472defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1473 1474multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1475 list<string> RtnModes = ["ret", "noret"]> { 1476 foreach RtnMode = RtnModes in { 1477 1478 defvar Op = !cast<SDPatternOperator>(OpPrefix 1479 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1480 1481 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1482 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1483 (set_glc $cachepolicy), (timm:$cachepolicy)); 1484 1485 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1486 def : GCNPat< 1487 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1488 timm:$offset, timm:$cachepolicy, 0)), 1489 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1490 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1491 timm:$offset, CachePolicy) 1492 >; 1493 1494 def : GCNPat< 1495 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1496 timm:$offset, timm:$cachepolicy, timm)), 1497 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1498 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1499 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1500 >; 1501 1502 def : GCNPat< 1503 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1504 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1505 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1506 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1507 SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1508 >; 1509 1510 def : GCNPat< 1511 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1512 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1513 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1514 getVregSrcForVT<vt>.ret:$vdata_in, 1515 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1516 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy) 1517 >; 1518 } // end let AddedComplexity 1519 1520 } // end foreach RtnMode 1521} 1522 1523defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1524defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1525defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1526defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1527defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1528defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1529defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1530defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1531defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1532defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1533defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1534defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1535defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1536defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1537defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1538defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1539defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1540defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1541defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1542defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1543defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1544defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1545defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1546defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1547defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1548defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1549 1550let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1551 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1552 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1553} 1554let SubtargetPredicate = isGFX6GFX7GFX10 in { 1555 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1556 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1557} 1558 1559class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1560 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1561 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7))> { 1562 let HasNoUse = true; 1563} 1564 1565multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1566 string opcode> { 1567 def : GCNPat< 1568 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1569 0, i32:$soffset, timm:$offset, 1570 timm:$cachepolicy, 0), 1571 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1572 timm:$offset, timm:$cachepolicy) 1573 >; 1574 1575 def : GCNPat< 1576 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1577 0, i32:$soffset, timm:$offset, 1578 timm:$cachepolicy, timm), 1579 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1580 timm:$offset, timm:$cachepolicy) 1581 >; 1582 1583 def : GCNPat< 1584 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1585 i32:$voffset, i32:$soffset, timm:$offset, 1586 timm:$cachepolicy, 0), 1587 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1588 timm:$offset, timm:$cachepolicy) 1589 >; 1590 1591 def : GCNPat< 1592 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1593 i32:$voffset, i32:$soffset, timm:$offset, 1594 timm:$cachepolicy, timm), 1595 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1596 getVregSrcForVT<vt>.ret:$vdata_in, 1597 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1598 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, timm:$cachepolicy) 1599 >; 1600} 1601 1602let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1603defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["noret"]>; 1604 1605let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16NoRtnInsts in 1606defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["noret"]>; 1607 1608let SubtargetPredicate = HasAtomicFaddRtnInsts in 1609defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32", ["ret"]>; 1610 1611let SubtargetPredicate = HasAtomicBufferGlobalPkAddF16Insts in 1612defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16", ["ret"]>; 1613 1614let SubtargetPredicate = isGFX90APlus in { 1615 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1616 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1617 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1618} // End SubtargetPredicate = isGFX90APlus 1619 1620foreach RtnMode = ["ret", "noret"] in { 1621 1622defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1623 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1624defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1625defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1626 (timm:$cachepolicy)); 1627 1628defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1629 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1630 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1631def : GCNPat< 1632 (Op 1633 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1634 timm:$offset, timm:$cachepolicy, 0), 1635 !if(!eq(RtnMode, "ret"), 1636 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1637 OffsetResDag) 1638>; 1639 1640defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1641 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1642 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1643 CachePolicy); 1644def : GCNPat< 1645 (Op 1646 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1647 0, i32:$soffset, timm:$offset, 1648 timm:$cachepolicy, timm), 1649 !if(!eq(RtnMode, "ret"), 1650 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1651 IdxenResDag) 1652>; 1653 1654defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1655 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1656 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1657 CachePolicy); 1658def : GCNPat< 1659 (Op 1660 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1661 i32:$voffset, i32:$soffset, timm:$offset, 1662 timm:$cachepolicy, 0), 1663 !if(!eq(RtnMode, "ret"), 1664 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1665 OffenResDag) 1666>; 1667 1668defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1669 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1670 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1671 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, CachePolicy); 1672def : GCNPat< 1673 (Op 1674 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1675 i32:$voffset, i32:$soffset, timm:$offset, 1676 timm:$cachepolicy, timm), 1677 !if(!eq(RtnMode, "ret"), 1678 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1679 BothenResDag) 1680>; 1681 1682} // end foreach RtnMode 1683 1684class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1685 PatFrag constant_ld> : GCNPat < 1686 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1687 i32:$offset))), 1688 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1689 >; 1690 1691multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1692 ValueType vt, PatFrag atomic_ld> { 1693 def : GCNPat < 1694 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), 1695 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1696 >; 1697 1698 def : GCNPat < 1699 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), 1700 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1701 >; 1702} 1703 1704let SubtargetPredicate = isGFX6GFX7 in { 1705def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1706def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1707def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1708def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1709def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1710def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1711 1712defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1713defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1714} // End SubtargetPredicate = isGFX6GFX7 1715 1716multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1717 PatFrag ld> { 1718 1719 def : GCNPat < 1720 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1721 (Instr_OFFSET $srsrc, $soffset, $offset) 1722 >; 1723} 1724 1725let OtherPredicates = [Has16BitInsts] in { 1726 1727defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1728defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1729defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1730defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1731defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1732defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1733 1734defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1735 1736} // End OtherPredicates = [Has16BitInsts] 1737 1738multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1739 MUBUF_Pseudo InstrOffset, 1740 ValueType vt, PatFrag ld> { 1741 def : GCNPat < 1742 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1743 i32:$soffset, i32:$offset))), 1744 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0) 1745 >; 1746 1747 def : GCNPat < 1748 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), 1749 (InstrOffset $srsrc, $soffset, $offset, 0, 0) 1750 >; 1751} 1752 1753// XXX - Is it possible to have a complex pattern in a PatFrag? 1754multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1755 MUBUF_Pseudo InstrOffset, 1756 ValueType vt, PatFrag ld_frag> { 1757 def : GCNPat < 1758 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, i32:$offset), vt:$in), 1759 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1760 >; 1761 1762 def : GCNPat < 1763 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$in), 1764 (InstrOffset $srsrc, $soffset, $offset, $in) 1765 >; 1766} 1767 1768let OtherPredicates = [DisableFlatScratch] in { 1769defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1770defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1771defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1772defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1773defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1774defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1775defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1776defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1777defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1778defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1779 1780foreach vt = Reg32Types.types in { 1781defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1782} 1783defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1784defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1785defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1786 1787let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1788defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1789defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1790defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1791defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1792defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1793defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1794 1795defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1796defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1797defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1798defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1799defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1800defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1801} 1802 1803} // End OtherPredicates = [DisableFlatScratch] 1804 1805multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1806 ValueType vt, PatFrag atomic_st> { 1807 // Store follows atomic op convention so address is first 1808 def : GCNPat < 1809 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), vt:$val), 1810 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1811 >; 1812 1813 def : GCNPat < 1814 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset), vt:$val), 1815 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1816 >; 1817} 1818let SubtargetPredicate = isGFX6GFX7 in { 1819defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1820defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1821defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1822defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1823defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1824defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1825} // End Predicates = isGFX6GFX7 1826 1827 1828multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1829 PatFrag st> { 1830 1831 def : GCNPat < 1832 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), 1833 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1834 >; 1835} 1836 1837defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1838defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1839 1840multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1841 MUBUF_Pseudo InstrOffset, 1842 ValueType vt, PatFrag st, 1843 RegisterClass rc = VGPR_32> { 1844 def : GCNPat < 1845 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1846 i32:$soffset, i32:$offset)), 1847 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) 1848 >; 1849 1850 def : GCNPat < 1851 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1852 i32:$offset)), 1853 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0) 1854 >; 1855} 1856 1857let OtherPredicates = [DisableFlatScratch] in { 1858defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1859defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1860defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1861defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1862 1863foreach vt = Reg32Types.types in { 1864defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1865} 1866 1867defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1868defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1869defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1870 1871 1872let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1873 // Hiding the extract high pattern in the PatFrag seems to not 1874 // automatically increase the complexity. 1875let AddedComplexity = 1 in { 1876defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1877defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1878} 1879} 1880} // End OtherPredicates = [DisableFlatScratch] 1881 1882//===----------------------------------------------------------------------===// 1883// MTBUF Patterns 1884//===----------------------------------------------------------------------===// 1885 1886//===----------------------------------------------------------------------===// 1887// tbuffer_load/store_format patterns 1888//===----------------------------------------------------------------------===// 1889 1890multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1891 string opcode, ValueType memoryVt = vt> { 1892 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1893 1894 def : GCNPat< 1895 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1896 timm:$format, timm:$auxiliary, 0)), 1897 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1898 (as_i8timm $format), 1899 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1900 >; 1901 1902 def : GCNPat< 1903 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1904 timm:$format, timm:$auxiliary, timm)), 1905 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1906 (as_i8timm $format), 1907 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1908 >; 1909 1910 def : GCNPat< 1911 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1912 timm:$format, timm:$auxiliary, 0)), 1913 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1914 (as_i8timm $format), 1915 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1916 >; 1917 1918 def : GCNPat< 1919 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1920 timm:$format, timm:$auxiliary, timm)), 1921 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1922 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1923 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, 1924 (as_i8timm $format), 1925 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1926 >; 1927} 1928 1929defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1930defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1931defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1932defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1933defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1934defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1935defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1936defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1937 1938let SubtargetPredicate = HasUnpackedD16VMem in { 1939 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1940 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1941 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1942 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1943 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1944} // End HasUnpackedD16VMem. 1945 1946let SubtargetPredicate = HasPackedD16VMem in { 1947 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1948 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1949 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1950 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1951 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1952} // End HasPackedD16VMem. 1953 1954multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1955 string opcode, ValueType memoryVt = vt> { 1956 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1957 1958 def : GCNPat< 1959 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1960 timm:$format, timm:$auxiliary, 0), 1961 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1962 timm:$offset, (as_i8timm $format), 1963 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1964 >; 1965 1966 def : GCNPat< 1967 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1968 timm:$format, timm:$auxiliary, timm), 1969 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1970 timm:$offset, (as_i8timm $format), 1971 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1972 >; 1973 1974 def : GCNPat< 1975 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1976 timm:$format, timm:$auxiliary, 0), 1977 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1978 timm:$offset, (as_i8timm $format), 1979 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1980 >; 1981 1982 def : GCNPat< 1983 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1984 timm:$offset, timm:$format, timm:$auxiliary, timm), 1985 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1986 getVregSrcForVT<vt>.ret:$vdata, 1987 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1988 SReg_128:$rsrc, SCSrc_b32:$soffset, timm:$offset, (as_i8timm $format), 1989 (extract_cpol $auxiliary), (extract_swz $auxiliary)) 1990 >; 1991} 1992 1993defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1994defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1995defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1996defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1997defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1998defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1999defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2000defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2001 2002let SubtargetPredicate = HasUnpackedD16VMem in { 2003 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2004 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2005 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2006 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2007 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2008} // End HasUnpackedD16VMem. 2009 2010let SubtargetPredicate = HasPackedD16VMem in { 2011 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2012 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2013 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2014 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2015 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2016} // End HasPackedD16VMem. 2017 2018//===----------------------------------------------------------------------===// 2019// Target-specific instruction encodings. 2020//===----------------------------------------------------------------------===// 2021 2022//===----------------------------------------------------------------------===// 2023// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2024//===----------------------------------------------------------------------===// 2025 2026class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2027 string real_name = ps.Mnemonic> : 2028 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2029 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2030 let Inst{31-26} = 0x38; 2031 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2032 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2033 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2034 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2035} 2036 2037class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2038 string real_name = ps.Mnemonic> : 2039 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2040 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2041 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2042 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2043 let Inst{25-18} = op; 2044 let Inst{53} = ps.tfe; 2045 let Inst{54} = ps.offen; 2046 let Inst{55} = ps.idxen; 2047} 2048 2049class Base_MUBUF_Real_Atomic_gfx11<bits<8> op, MUBUF_Pseudo ps, 2050 string real_name> : 2051 MUBUF_Real_gfx11<op, ps, real_name> { 2052 let Inst{13} = cpol{CPolBit.DLC}; 2053} 2054 2055class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2056 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2057 let Inst{12} = ps.offen; 2058 let Inst{13} = ps.idxen; 2059 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2060 let Inst{16} = ps.lds; 2061 let Inst{24-18} = op; 2062 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2063 let Inst{55} = ps.tfe; 2064} 2065 2066class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2067 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2068 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2069 let Inst{25} = op{7}; 2070} 2071 2072class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2073 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2074 let Inst{15} = ps.addr64; 2075} 2076 2077//===----------------------------------------------------------------------===// 2078// MUBUF - GFX11. 2079//===----------------------------------------------------------------------===// 2080 2081// Shortcut to default Mnemonic from MUBUF_Pseudo. Hides the cast to the 2082// specific pseudo (bothen in this case) since any of them will work. 2083class get_MUBUF_ps<string name> { 2084 string Mnemonic = !cast<MUBUF_Pseudo>(name # "_BOTHEN").Mnemonic; 2085} 2086 2087// gfx11 instruction that accept both old and new assembler name. 2088class Pre_gfx11_MUBUF_Name <string mnemonic, string real_name> : 2089 MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>; 2090 2091class MUBUF_Real_gfx11_impl<bits<8> op, string ps_name, string real_name> : 2092 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2093let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2094multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<bits<8> op, string real_name> { 2095 def _BOTHEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_BOTHEN", real_name>; 2096 def _IDXEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_IDXEN", real_name>; 2097 def _OFFEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFEN", real_name>; 2098 def _OFFSET_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFSET", real_name>; 2099} 2100 2101multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name, 2102 bit hasTFE = 1> { 2103 defm NAME : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2104 if hasTFE then 2105 defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>; 2106} 2107 2108// Non-renamed, non-atomic gfx11 mubuf instructions. 2109multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> : 2110 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>; 2111 2112multiclass MUBUF_Real_AllAddr_gfx11_Renamed<bits<8> op, string real_name> : 2113 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2114 def : Pre_gfx11_MUBUF_Name<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2115} 2116 2117class MUBUF_Real_Atomic_gfx11_impl<bits<8> op, string ps_name, 2118 string real_name> : 2119 Base_MUBUF_Real_Atomic_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>; 2120let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2121multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return, 2122 string real_name> { 2123 defvar Rtn = !if(!eq(is_return, 1), "_RTN", ""); 2124 def _BOTHEN#Rtn#_gfx11 : 2125 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_BOTHEN" # Rtn, real_name>, 2126 AtomicNoRet<NAME # "_BOTHEN_gfx11", is_return>; 2127 def _IDXEN#Rtn#_gfx11 : 2128 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_IDXEN" # Rtn, real_name>, 2129 AtomicNoRet<NAME # "_IDXEN_gfx11", is_return>; 2130 def _OFFEN#Rtn#_gfx11 : 2131 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFEN" # Rtn, real_name>, 2132 AtomicNoRet<NAME # "_OFFEN_gfx11", is_return>; 2133 def _OFFSET#Rtn#_gfx11 : 2134 MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFSET" # Rtn, real_name>, 2135 AtomicNoRet<NAME # "_OFFSET_gfx11", is_return>; 2136} 2137 2138// Non-renamed gfx11 mubuf atomic. 2139multiclass MUBUF_Real_Atomic_gfx11<bits<8> op> : 2140 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>, 2141 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>; 2142 2143multiclass MUBUF_Real_Atomic_gfx11_Renamed<bits<8> op, string real_name> : 2144 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, real_name>, 2145 MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, real_name> { 2146 def : Pre_gfx11_MUBUF_Name<get_MUBUF_ps<NAME>.Mnemonic, real_name>; 2147} 2148 2149let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2150def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2151def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2152} 2153 2154defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x014, "buffer_load_b32">; 2155defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x015, "buffer_load_b64">; 2156defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x016, "buffer_load_b96">; 2157defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x017, "buffer_load_b128">; 2158defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x020, "buffer_load_d16_b16">; 2159defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x008, "buffer_load_d16_format_x">; 2160defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x009, "buffer_load_d16_format_xy">; 2161defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2162defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2163defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x023, "buffer_load_d16_hi_b16">; 2164defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2165defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x022, "buffer_load_d16_hi_i8">; 2166defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x021, "buffer_load_d16_hi_u8">; 2167defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01f, "buffer_load_d16_i8">; 2168defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01e, "buffer_load_d16_u8">; 2169defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x000>; 2170defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x001>; 2171defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x002>; 2172defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x003>; 2173defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x011, "buffer_load_i8">; 2174defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x013, "buffer_load_i16">; 2175defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x010, "buffer_load_u8">; 2176defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x012, "buffer_load_u16">; 2177defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031, 0>; 2178defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032, 0>; 2179defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e, 0>; 2180defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030, 0>; 2181defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d, 0>; 2182defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f, 0>; 2183defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x018, "buffer_store_b8">; 2184defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x019, "buffer_store_b16">; 2185defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x01A, "buffer_store_b32">; 2186defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01B, "buffer_store_b64">; 2187defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01C, "buffer_store_b96">; 2188defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01D, "buffer_store_b128">; 2189defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x00C, "buffer_store_d16_format_x">; 2190defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x00D, "buffer_store_d16_format_xy">; 2191defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2192defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2193defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x024, "buffer_store_d16_hi_b8">; 2194defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x025, "buffer_store_d16_hi_b16">; 2195defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2196defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x004>; 2197defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x005>; 2198defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x006>; 2199defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x007>; 2200defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_gfx11<0x056>; 2201defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_gfx11_Renamed<0x035, "buffer_atomic_add_u32">; 2202defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x043, "buffer_atomic_add_u64">; 2203defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_gfx11_Renamed<0x03C, "buffer_atomic_and_b32">; 2204defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x049, "buffer_atomic_and_b64">; 2205defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2206defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2207defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2208defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomic_gfx11_Renamed_impl<0x037, 1, "buffer_atomic_csub_u32">; 2209def : Pre_gfx11_MUBUF_Name<"buffer_atomic_csub", "buffer_atomic_csub_u32">; 2210defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">; 2211defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">; 2212defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_gfx11_Renamed<0x03F, "buffer_atomic_inc_u32">; 2213defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04C, "buffer_atomic_inc_u64">; 2214defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_gfx11_Renamed<0x052, "buffer_atomic_max_f32">; 2215defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_gfx11_Renamed<0x03A, "buffer_atomic_max_i32">; 2216defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x047, "buffer_atomic_max_i64">; 2217defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_gfx11_Renamed<0x03B, "buffer_atomic_max_u32">; 2218defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x048, "buffer_atomic_max_u64">; 2219defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_gfx11_Renamed<0x051, "buffer_atomic_min_f32">; 2220defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_gfx11_Renamed<0x038, "buffer_atomic_min_i32">; 2221defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x045, "buffer_atomic_min_i64">; 2222defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_gfx11_Renamed<0x039, "buffer_atomic_min_u32">; 2223defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x046, "buffer_atomic_min_u64">; 2224defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_gfx11_Renamed<0x03D, "buffer_atomic_or_b32">; 2225defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04A, "buffer_atomic_or_b64">; 2226defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_gfx11_Renamed<0x036, "buffer_atomic_sub_u32">; 2227defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x044, "buffer_atomic_sub_u64">; 2228defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_gfx11_Renamed<0x033, "buffer_atomic_swap_b32">; 2229defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x041, "buffer_atomic_swap_b64">; 2230defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_gfx11_Renamed<0x03E, "buffer_atomic_xor_b32">; 2231defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_gfx11_Renamed<0x04B, "buffer_atomic_xor_b64">; 2232 2233//===----------------------------------------------------------------------===// 2234// MUBUF - GFX10. 2235//===----------------------------------------------------------------------===// 2236 2237let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2238 multiclass MUBUF_Real_AllAddr_Helper_gfx10<bits<8> op> { 2239 def _BOTHEN_gfx10 : 2240 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2241 def _IDXEN_gfx10 : 2242 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2243 def _OFFEN_gfx10 : 2244 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2245 def _OFFSET_gfx10 : 2246 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2247 } 2248 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2249 defm NAME : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2250 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx10<op>; 2251 } 2252 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op, bit isTFE = 0> { 2253 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2254 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2255 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2256 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2257 2258 if !not(isTFE) then { 2259 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2260 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2261 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2262 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2263 } 2264 } 2265 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2266 def _BOTHEN_RTN_gfx10 : 2267 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2268 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2269 def _IDXEN_RTN_gfx10 : 2270 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2271 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2272 def _OFFEN_RTN_gfx10 : 2273 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2274 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2275 def _OFFSET_RTN_gfx10 : 2276 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2277 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2278 } 2279 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2280 MUBUF_Real_Atomics_RTN_gfx10<op> { 2281 def _BOTHEN_gfx10 : 2282 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2283 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2284 def _IDXEN_gfx10 : 2285 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2286 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2287 def _OFFEN_gfx10 : 2288 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2289 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2290 def _OFFSET_gfx10 : 2291 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2292 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2293 } 2294} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2295 2296defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2297defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2298defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2299defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2300defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2301defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2302defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2303defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2304// FIXME-GFX10: Add following instructions: 2305//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2306//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2307defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2308defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2309defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2310defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2311defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2312defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2313defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2314defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2315 2316def BUFFER_GL0_INV_gfx10 : 2317 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2318def BUFFER_GL1_INV_gfx10 : 2319 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2320 2321//===----------------------------------------------------------------------===// 2322// MUBUF - GFX6, GFX7, GFX10. 2323//===----------------------------------------------------------------------===// 2324 2325let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2326 multiclass MUBUF_Real_gfx6<bits<8> op> { 2327 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2328 } 2329} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2330 2331let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2332 multiclass MUBUF_Real_gfx7<bits<8> op> { 2333 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2334 } 2335} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2336 2337let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2338 multiclass MUBUF_Real_AllAddr_Helper_gfx6_gfx7<bits<8> op> { 2339 def _ADDR64_gfx6_gfx7 : 2340 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2341 def _BOTHEN_gfx6_gfx7 : 2342 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2343 def _IDXEN_gfx6_gfx7 : 2344 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2345 def _OFFEN_gfx6_gfx7 : 2346 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2347 def _OFFSET_gfx6_gfx7 : 2348 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2349 } 2350 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2351 defm NAME : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2352 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx6_gfx7<op>; 2353 } 2354 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op, bit isTFE = 0> { 2355 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2356 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2357 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2358 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2359 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2360 2361 if !not(isTFE) then { 2362 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2363 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2364 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2365 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2366 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2367 } 2368 } 2369 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2370 def _ADDR64_gfx6_gfx7 : 2371 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2372 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2373 def _BOTHEN_gfx6_gfx7 : 2374 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2375 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2376 def _IDXEN_gfx6_gfx7 : 2377 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2378 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2379 def _OFFEN_gfx6_gfx7 : 2380 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2381 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2382 def _OFFSET_gfx6_gfx7 : 2383 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2384 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2385 2386 def _ADDR64_RTN_gfx6_gfx7 : 2387 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2388 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2389 def _BOTHEN_RTN_gfx6_gfx7 : 2390 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2391 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2392 def _IDXEN_RTN_gfx6_gfx7 : 2393 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2394 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2395 def _OFFEN_RTN_gfx6_gfx7 : 2396 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2397 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2398 def _OFFSET_RTN_gfx6_gfx7 : 2399 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2400 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2401 } 2402} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2403 2404multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2405 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2406 2407multiclass MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<bits<8> op, bit isTFE = 0> : 2408 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op, isTFE>, 2409 MUBUF_Real_AllAddr_Lds_gfx10<op, isTFE>; 2410 2411multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> { 2412 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op>; 2413 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_gfx6_gfx7_gfx10<op, 1>; 2414} 2415 2416multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2417 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2418 2419// FIXME-GFX6: Following instructions are available only on GFX6. 2420//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2421//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2422 2423defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2424defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2425defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2426defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2427defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2428defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2429defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2430defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2431defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2432defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2433defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2434defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2435defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2436defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2437defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2438defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2439defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2440defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2441defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2442defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2443defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2444defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2445 2446defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2447defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2448defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2449defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2450defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2451defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2452defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2453defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2454defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2455defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2456defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2457defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2458defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2459defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2460defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2461defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2462defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2463defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2464defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2465defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2466defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2467defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2468defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2469defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2470defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2471defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2472defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2473defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2474defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2475// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2476defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2477defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2478defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2479 2480defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2481 2482defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2483defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2484def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2485 2486//===----------------------------------------------------------------------===// 2487// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2488//===----------------------------------------------------------------------===// 2489 2490class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2491 string real_name = ps.Mnemonic> : 2492 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2493 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2494 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2495 let Inst{31-26} = 0x3a; //encoding 2496 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2497 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2498 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2499 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2500} 2501 2502class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2503 string real_name = ps.Mnemonic> : 2504 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2505 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2506 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2507 let Inst{18-15} = op; 2508 let Inst{25-19} = format; 2509 let Inst{53} = ps.tfe; 2510 let Inst{54} = ps.offen; 2511 let Inst{55} = ps.idxen; 2512} 2513 2514class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2515 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2516 let Inst{12} = ps.offen; 2517 let Inst{13} = ps.idxen; 2518 let Inst{18-16} = op; 2519 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2520 let Inst{55} = ps.tfe; 2521} 2522 2523//===----------------------------------------------------------------------===// 2524// MTBUF - GFX11. 2525//===----------------------------------------------------------------------===// 2526 2527let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2528multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> { 2529 def _BOTHEN_gfx11 : 2530 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2531 def _IDXEN_gfx11 : 2532 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2533 def _OFFEN_gfx11 : 2534 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2535 def _OFFSET_gfx11 : 2536 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2537} 2538 2539multiclass MTBUF_Real_AllAddr_gfx11_Impl<bits<4> op, MTBUF_Pseudo ps> 2540 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2541multiclass MTBUF_Real_AllAddr_gfx11<bits<4> op> 2542 : MTBUF_Real_AllAddr_gfx11_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2543 2544 2545class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2546 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2547multiclass MTBUF_Real_AllAddr_gfx11_Renamed<bits<4> op, string real_name> 2548 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2549 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2550} 2551 2552defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x008, "tbuffer_load_d16_format_x">; 2553defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2554defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 2555defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 2556defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x000>; 2557defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x001>; 2558defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x002>; 2559defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x003>; 2560defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x00c, "tbuffer_store_d16_format_x">; 2561defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 2562defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 2563defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 2564defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x004>; 2565defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x005>; 2566defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x006>; 2567defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x007>; 2568 2569//===----------------------------------------------------------------------===// 2570// MTBUF - GFX10. 2571//===----------------------------------------------------------------------===// 2572 2573class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2574 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2575 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2576 let Inst{25-19} = format; 2577 let Inst{53} = op{3}; 2578} 2579 2580let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2581 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2582 def _BOTHEN_gfx10 : 2583 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2584 def _IDXEN_gfx10 : 2585 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2586 def _OFFEN_gfx10 : 2587 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2588 def _OFFSET_gfx10 : 2589 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2590 } 2591} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2592 2593defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2594defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2595defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2596defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2597defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2598defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2599defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2600defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2601 2602//===----------------------------------------------------------------------===// 2603// MTBUF - GFX6, GFX7, GFX10. 2604//===----------------------------------------------------------------------===// 2605 2606class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2607 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2608 let Inst{15} = ps.addr64; 2609 let Inst{22-19} = dfmt; 2610 let Inst{25-23} = nfmt; 2611} 2612 2613let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2614 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2615 def _ADDR64_gfx6_gfx7 : 2616 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2617 def _BOTHEN_gfx6_gfx7 : 2618 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2619 def _IDXEN_gfx6_gfx7 : 2620 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2621 def _OFFEN_gfx6_gfx7 : 2622 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2623 def _OFFSET_gfx6_gfx7 : 2624 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2625 } 2626} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2627 2628multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2629 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2630 2631defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2632defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2633defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2634defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2635defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2636defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2637defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2638defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2639 2640//===----------------------------------------------------------------------===// 2641// GFX8, GFX9 (VI). 2642//===----------------------------------------------------------------------===// 2643 2644class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2645 bit has_sccb = ps.has_sccb> : 2646 MUBUF_Real<ps>, 2647 Enc64, 2648 SIMCInstr<ps.PseudoInstr, Enc>, 2649 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2650 !if(ps.IsAtomicRet, 1, ?))> { 2651 2652 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2653 let Inst{12} = ps.offen; 2654 let Inst{13} = ps.idxen; 2655 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2656 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2657 let Inst{16} = ps.lds; 2658 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2659 let Inst{24-18} = op; 2660 let Inst{31-26} = 0x38; //encoding 2661 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2662 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2663 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2664 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2665} 2666 2667class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2668 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2669 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2670 let DecoderNamespace = "GFX8"; 2671 2672 let Inst{55} = ps.tfe; 2673} 2674 2675class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2676 bit has_sccb = ps.has_sccb> : 2677 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2678 let AssemblerPredicate = isGFX90APlus; 2679 let DecoderNamespace = "GFX90A"; 2680 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2681 ps.AsmOperands); 2682 2683 let Inst{55} = acc; 2684} 2685 2686class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2687 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2688 let AssemblerPredicate = isGFX940Plus; 2689 let DecoderNamespace = "GFX9"; 2690 let AsmString = ps.Mnemonic # ps.AsmOperands; 2691 2692 let Inst{55} = acc; 2693} 2694 2695multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps, bit isTFE = 0> { 2696 def _vi : MUBUF_Real_vi<op, ps>; 2697 2698 if !not(isTFE) then { 2699 if !not(ps.FPAtomic) then 2700 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2701 } 2702 2703 if ps.FPAtomic then { 2704 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2705 let SubtargetPredicate = isGFX90AOnly; 2706 let AssemblerPredicate = isGFX90AOnly; 2707 } 2708 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2709 } 2710} 2711 2712multiclass MUBUF_Real_AllAddr_Helper_vi<bits<7> op, bit isTFE = 0> { 2713 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), isTFE>; 2714 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), isTFE>; 2715 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), isTFE>; 2716 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), isTFE>; 2717} 2718 2719multiclass MUBUF_Real_AllAddr_vi<bits<7> op, bit hasTFE = 1> { 2720 defm NAME : MUBUF_Real_AllAddr_Helper_vi<op>; 2721 if hasTFE then 2722 defm _TFE : MUBUF_Real_AllAddr_Helper_vi<op, 1>; 2723} 2724 2725multiclass MUBUF_Real_AllAddr_Lds_Helper_vi<bits<7> op, bit isTFE = 0> { 2726 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2727 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2728 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2729 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2730 2731 if !not(isTFE) then { 2732 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2733 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2734 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2735 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2736 2737 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2738 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2739 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2740 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2741 2742 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2743 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2744 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2745 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2746 } 2747} 2748 2749multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2750 defm NAME : MUBUF_Real_AllAddr_Lds_Helper_vi<op>; 2751 defm _TFE : MUBUF_Real_AllAddr_Lds_Helper_vi<op, 1>; 2752} 2753 2754class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2755 MUBUF_Real<ps>, 2756 Enc64, 2757 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2758 let AssemblerPredicate=HasUnpackedD16VMem; 2759 let DecoderNamespace="GFX80_UNPACKED"; 2760 2761 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2762 let Inst{12} = ps.offen; 2763 let Inst{13} = ps.idxen; 2764 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2765 let Inst{16} = ps.lds; 2766 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2767 let Inst{24-18} = op; 2768 let Inst{31-26} = 0x38; //encoding 2769 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2770 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2771 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2772 let Inst{55} = ps.tfe; 2773 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2774} 2775 2776multiclass MUBUF_Real_AllAddr_Helper_gfx80<bits<7> op> { 2777 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2778 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2779 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2780 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2781} 2782 2783multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2784 defm NAME : MUBUF_Real_AllAddr_Helper_gfx80<op>; 2785 defm _TFE : MUBUF_Real_AllAddr_Helper_gfx80<op>; 2786} 2787 2788multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2789 MUBUF_Real_AllAddr_vi<op, 0> { 2790 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2791 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2792 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2793 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2794} 2795 2796defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2797defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2798defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2799defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2800defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2801defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2802defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2803defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2804let SubtargetPredicate = HasUnpackedD16VMem in { 2805 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2806 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2807 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2808 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2809 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2810 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2811 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2812 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2813} // End HasUnpackedD16VMem. 2814let SubtargetPredicate = HasPackedD16VMem in { 2815 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2816 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2817 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2818 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2819 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2820 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2821 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2822 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2823} // End HasPackedD16VMem. 2824defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2825defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2826defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2827defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2828defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2829defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 2830defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 2831defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 2832defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2833defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2834defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2835defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2836defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2837defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2838defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2839defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2840 2841defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2842defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2843defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2844defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2845defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2846defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2847 2848defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2849defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2850 2851defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2852defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2853defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2854defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2855defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2856defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2857defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2858defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2859defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2860defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2861defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2862defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2863defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2864 2865defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2866defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2867defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2868defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2869defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2870defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2871defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2872defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2873defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2874defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2875defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2876defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2877defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2878 2879defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2880 2881let AssemblerPredicate = isGFX8GFX9 in { 2882def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2883def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2884} // End AssemblerPredicate = isGFX8GFX9 2885 2886 2887defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2888 2889let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 2890defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2891} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 2892 2893let SubtargetPredicate = isGFX90APlus in { 2894 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2895 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2896 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2897} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2898 2899def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2900 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2901 let AssemblerPredicate = isGFX90AOnly; 2902 let SubtargetPredicate = isGFX90AOnly; 2903} 2904def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2905 2906let SubtargetPredicate = isGFX940Plus in { 2907def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2908def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2909} 2910 2911class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2912 MTBUF_Real<ps>, 2913 Enc64, 2914 SIMCInstr<ps.PseudoInstr, Enc> { 2915 2916 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2917 let Inst{12} = ps.offen; 2918 let Inst{13} = ps.idxen; 2919 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2920 let Inst{18-15} = op; 2921 let Inst{22-19} = dfmt; 2922 let Inst{25-23} = nfmt; 2923 let Inst{31-26} = 0x3a; //encoding 2924 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2925 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2926 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2927 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2928 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2929 let Inst{55} = ps.tfe; 2930 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2931} 2932 2933class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2934 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2935 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2936 let DecoderNamespace = "GFX8"; 2937 2938 let Inst{55} = ps.tfe; 2939} 2940 2941class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2942 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2943 let AssemblerPredicate = isGFX90APlus; 2944 let DecoderNamespace = "GFX90A"; 2945 let AsmString = ps.Mnemonic # ps.AsmOperands; 2946 2947 let Inst{55} = acc; 2948} 2949 2950multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2951 def _vi : MTBUF_Real_vi<op, ps>; 2952 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2953} 2954 2955multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2956 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2957 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2958 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2959 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2960} 2961 2962class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2963 MTBUF_Real<ps>, 2964 Enc64, 2965 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2966 let AssemblerPredicate=HasUnpackedD16VMem; 2967 let DecoderNamespace="GFX80_UNPACKED"; 2968 2969 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2970 let Inst{12} = ps.offen; 2971 let Inst{13} = ps.idxen; 2972 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2973 let Inst{18-15} = op; 2974 let Inst{22-19} = dfmt; 2975 let Inst{25-23} = nfmt; 2976 let Inst{31-26} = 0x3a; //encoding 2977 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2978 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2979 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2980 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2981 let Inst{55} = ps.tfe; 2982 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2983} 2984 2985multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2986 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2987 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2988 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2989 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2990} 2991 2992defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2993defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2994defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2995defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2996defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2997defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2998defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2999defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 3000let SubtargetPredicate = HasUnpackedD16VMem in { 3001 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 3002 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 3003 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 3004 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 3005 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 3006 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 3007 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 3008 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 3009} // End HasUnpackedD16VMem. 3010let SubtargetPredicate = HasPackedD16VMem in { 3011 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 3012 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 3013 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 3014 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 3015 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 3016 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 3017 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 3018 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 3019} // End HasUnpackedD16VMem. 3020 3021def MUBUFInfoTable : GenericTable { 3022 let FilterClass = "MUBUF_Pseudo"; 3023 let CppTypeName = "MUBUFInfo"; 3024 let Fields = [ 3025 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3026 "IsBufferInv" 3027 ]; 3028 3029 let PrimaryKey = ["Opcode"]; 3030 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3031} 3032 3033def getMUBUFInfoFromOpcode : SearchIndex { 3034 let Table = MUBUFInfoTable; 3035 let Key = ["Opcode"]; 3036} 3037 3038def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3039 let Table = MUBUFInfoTable; 3040 let Key = ["BaseOpcode", "elements"]; 3041} 3042 3043def MTBUFInfoTable : GenericTable { 3044 let FilterClass = "MTBUF_Pseudo"; 3045 let CppTypeName = "MTBUFInfo"; 3046 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3047 3048 let PrimaryKey = ["Opcode"]; 3049 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3050} 3051 3052def getMTBUFInfoFromOpcode : SearchIndex { 3053 let Table = MTBUFInfoTable; 3054 let Key = ["Opcode"]; 3055} 3056 3057def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3058 let Table = MTBUFInfoTable; 3059 let Key = ["BaseOpcode", "elements"]; 3060} 3061