1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr64 : ComplexPattern<iPTR, 4, "SelectMUBUFAddr64">; 10def MUBUFOffset : ComplexPattern<iPTR, 3, "SelectMUBUFOffset">; 11 12def MUBUFScratchOffen : ComplexPattern<iPTR, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 13def MUBUFScratchOffset : ComplexPattern<iPTR, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 14 15def BUFAddrKind { 16 int Offset = 0; 17 int OffEn = 1; 18 int IdxEn = 2; 19 int BothEn = 3; 20 int Addr64 = 4; 21} 22 23class getAddrName<int addrKind> { 24 string ret = 25 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 26 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 27 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 28 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 29 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 30 ""))))); 31} 32 33class MUBUFAddr64Table <bit is_addr64, string Name> { 34 bit IsAddr64 = is_addr64; 35 string OpName = Name; 36} 37 38class MTBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43//===----------------------------------------------------------------------===// 44// MTBUF classes 45//===----------------------------------------------------------------------===// 46 47class MTBUFGetBaseOpcode<string Op> { 48 string ret = !subst("FORMAT_XY", "FORMAT_X", 49 !subst("FORMAT_XYZ", "FORMAT_X", 50 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 51} 52 53 54class MTBUF_Pseudo <string opName, dag outs, dag ins, 55 string asmOps, list<dag> pattern=[]> : 56 InstSI<outs, ins, "", pattern>, 57 SIMCInstr<opName, SIEncodingFamily.NONE> { 58 59 let isPseudo = 1; 60 let isCodeGenOnly = 1; 61 let Size = 8; 62 let UseNamedOperandTable = 1; 63 64 string Mnemonic = opName; 65 string AsmOperands = asmOps; 66 67 Instruction Opcode = !cast<Instruction>(NAME); 68 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 69 70 let VM_CNT = 1; 71 let EXP_CNT = 1; 72 let MTBUF = 1; 73 let Uses = [EXEC]; 74 let hasSideEffects = 0; 75 let SchedRW = [WriteVMEM]; 76 77 let AsmMatchConverter = "cvtMtbuf"; 78 79 bits<1> offen = 0; 80 bits<1> idxen = 0; 81 bits<1> addr64 = 0; 82 bits<1> has_vdata = 1; 83 bits<1> has_vaddr = 1; 84 bits<1> has_glc = 1; 85 bits<1> has_dlc = 1; 86 bits<1> glc_value = 0; // the value for glc if no such operand 87 bits<1> dlc_value = 0; // the value for dlc if no such operand 88 bits<1> has_srsrc = 1; 89 bits<1> has_soffset = 1; 90 bits<1> has_offset = 1; 91 bits<1> has_slc = 1; 92 bits<1> has_tfe = 1; 93 bits<4> elements = 0; 94 bits<1> has_sccb = 1; 95 bits<1> sccb_value = 0; 96} 97 98class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> : 99 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 100 101 let isPseudo = 0; 102 let isCodeGenOnly = 0; 103 104 let VM_CNT = 1; 105 let EXP_CNT = 1; 106 let MTBUF = 1; 107 108 // copy relevant pseudo op flags 109 let UseNamedOperandTable = ps.UseNamedOperandTable; 110 let SubtargetPredicate = ps.SubtargetPredicate; 111 let AsmMatchConverter = ps.AsmMatchConverter; 112 let Constraints = ps.Constraints; 113 let DisableEncoding = ps.DisableEncoding; 114 let TSFlags = ps.TSFlags; 115 let SchedRW = ps.SchedRW; 116 let mayLoad = ps.mayLoad; 117 let mayStore = ps.mayStore; 118 let IsAtomicRet = ps.IsAtomicRet; 119 let IsAtomicNoRet = ps.IsAtomicNoRet; 120 121 bits<12> offset; 122 bits<5> cpol; 123 bits<7> format; 124 bits<8> vaddr; 125 bits<10> vdata; 126 bits<7> srsrc; 127 bits<1> tfe; 128 bits<8> soffset; 129 130 bits<4> dfmt = format{3-0}; 131 bits<3> nfmt = format{6-4}; 132 133 // GFX90A+ only: instruction uses AccVGPR for data 134 // Bit supersedes tfe. 135 bits<1> acc = !if(ps.has_vdata, vdata{9}, 0); 136} 137 138class getMTBUFInsDA<list<RegisterClass> vdataList, 139 list<RegisterClass> vaddrList=[]> { 140 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 141 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 142 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 143 dag InsNoData = !if(!empty(vaddrList), 144 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 145 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz), 146 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 147 offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz) 148 ); 149 dag InsData = !if(!empty(vaddrList), 150 (ins vdata_op:$vdata, SReg_128:$srsrc, 151 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 152 TFE:$tfe, SWZ:$swz), 153 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 154 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, 155 TFE:$tfe, SWZ:$swz) 156 ); 157 dag ret = !if(!empty(vdataList), InsNoData, InsData); 158} 159 160class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 161 dag ret = 162 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 163 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 (ins)))))); 168} 169 170class getMTBUFAsmOps<int addrKind> { 171 string Pfx = 172 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 173 !if(!eq(addrKind, BUFAddrKind.OffEn), 174 "$vaddr, $srsrc,$format $soffset offen", 175 !if(!eq(addrKind, BUFAddrKind.IdxEn), 176 "$vaddr, $srsrc,$format $soffset idxen", 177 !if(!eq(addrKind, BUFAddrKind.BothEn), 178 "$vaddr, $srsrc,$format $soffset idxen offen", 179 !if(!eq(addrKind, BUFAddrKind.Addr64), 180 "$vaddr, $srsrc,$format $soffset addr64", 181 ""))))); 182 string ret = Pfx # "$offset"; 183} 184 185class MTBUF_SetupAddr<int addrKind> { 186 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 187 !eq(addrKind, BUFAddrKind.BothEn)); 188 189 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 190 !eq(addrKind, BUFAddrKind.BothEn)); 191 192 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 193 194 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 195} 196 197class MTBUF_Load_Pseudo <string opName, 198 int addrKind, 199 RegisterClass vdataClass, 200 int elems, 201 list<dag> pattern=[], 202 // Workaround bug bz30254 203 int addrKindCopy = addrKind> 204 : MTBUF_Pseudo<opName, 205 (outs getLdStRegisterOperand<vdataClass>.ret:$vdata), 206 getMTBUFIns<addrKindCopy>.ret, 207 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 208 pattern>, 209 MTBUF_SetupAddr<addrKindCopy> { 210 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 211 let mayLoad = 1; 212 let mayStore = 0; 213 let elements = elems; 214} 215 216multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 217 int elems> { 218 219 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 220 MTBUFAddr64Table<0, NAME>; 221 222 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 223 MTBUFAddr64Table<1, NAME>; 224 225 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 226 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 227 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 228 229 let DisableWQM = 1 in { 230 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 231 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 232 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 233 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 234 } 235} 236 237class MTBUF_Store_Pseudo <string opName, 238 int addrKind, 239 RegisterClass vdataClass, 240 int elems, 241 list<dag> pattern=[], 242 // Workaround bug bz30254 243 int addrKindCopy = addrKind, 244 RegisterClass vdataClassCopy = vdataClass> 245 : MTBUF_Pseudo<opName, 246 (outs), 247 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 248 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 249 pattern>, 250 MTBUF_SetupAddr<addrKindCopy> { 251 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 252 let mayLoad = 0; 253 let mayStore = 1; 254 let elements = elems; 255} 256 257multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 258 int elems> { 259 260 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>, 261 MTBUFAddr64Table<0, NAME>; 262 263 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>, 264 MTBUFAddr64Table<1, NAME>; 265 266 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 267 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 268 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 269 270 let DisableWQM = 1 in { 271 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 272 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 273 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 274 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 275 } 276} 277 278 279//===----------------------------------------------------------------------===// 280// MUBUF classes 281//===----------------------------------------------------------------------===// 282 283class MUBUFGetBaseOpcode<string Op> { 284 string ret = !subst("DWORDX2", "DWORD", 285 !subst("DWORDX3", "DWORD", 286 !subst("DWORDX4", "DWORD", Op))); 287} 288 289class MUBUF_Pseudo <string opName, dag outs, dag ins, 290 string asmOps, list<dag> pattern=[]> : 291 InstSI<outs, ins, "", pattern>, 292 SIMCInstr<opName, SIEncodingFamily.NONE> { 293 294 let isPseudo = 1; 295 let isCodeGenOnly = 1; 296 let Size = 8; 297 let UseNamedOperandTable = 1; 298 299 string Mnemonic = opName; 300 string AsmOperands = asmOps; 301 302 Instruction Opcode = !cast<Instruction>(NAME); 303 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 304 305 let VM_CNT = 1; 306 let EXP_CNT = 1; 307 let MUBUF = 1; 308 let Uses = [EXEC]; 309 let hasSideEffects = 0; 310 let SchedRW = [WriteVMEM]; 311 312 let AsmMatchConverter = "cvtMubuf"; 313 314 bits<1> offen = 0; 315 bits<1> idxen = 0; 316 bits<1> addr64 = 0; 317 bits<1> lds = 0; 318 bits<1> has_vdata = !not(lds); 319 bits<1> has_vaddr = 1; 320 bits<1> has_glc = 1; 321 bits<1> has_dlc = 1; 322 bits<1> glc_value = 0; // the value for glc if no such operand 323 bits<1> dlc_value = 0; // the value for dlc if no such operand 324 bits<1> has_srsrc = 1; 325 bits<1> has_soffset = 1; 326 bits<1> has_offset = 1; 327 bits<1> has_slc = 1; 328 bits<1> has_tfe = 1; 329 bits<4> elements = 0; 330 bits<1> has_sccb = 1; 331 bits<1> sccb_value = 0; 332 bits<1> IsBufferInv = 0; 333} 334 335class MUBUF_Real <MUBUF_Pseudo ps, string real_name = ps.Mnemonic> : 336 InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []> { 337 338 let isPseudo = 0; 339 let isCodeGenOnly = 0; 340 341 let VM_CNT = 1; 342 let EXP_CNT = 1; 343 let MUBUF = 1; 344 345 // copy relevant pseudo op flags 346 let SubtargetPredicate = ps.SubtargetPredicate; 347 let AsmMatchConverter = ps.AsmMatchConverter; 348 let OtherPredicates = ps.OtherPredicates; 349 let Constraints = ps.Constraints; 350 let DisableEncoding = ps.DisableEncoding; 351 let TSFlags = ps.TSFlags; 352 let UseNamedOperandTable = ps.UseNamedOperandTable; 353 let SchedRW = ps.SchedRW; 354 let mayLoad = ps.mayLoad; 355 let mayStore = ps.mayStore; 356 let IsAtomicRet = ps.IsAtomicRet; 357 let IsAtomicNoRet = ps.IsAtomicNoRet; 358 let VALU = ps.VALU; 359 let LGKM_CNT = ps.LGKM_CNT; 360 361 bits<12> offset; 362 bits<5> cpol; 363 bits<8> vaddr; 364 bits<10> vdata; 365 bits<7> srsrc; 366 bits<1> tfe; 367 bits<8> soffset; 368 369 // GFX90A+ only: instruction uses AccVGPR for data 370 // Bit supersedes tfe. 371 bits<1> acc = !if(ps.has_vdata, vdata{9}, !if(ps.lds, ?, 0)); 372} 373 374 375// For cache invalidation instructions. 376class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 377 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 378 379 let AsmMatchConverter = ""; 380 381 let hasSideEffects = 1; 382 let mayLoad = 0; 383 let mayStore = 0; 384 385 let IsBufferInv = 1; 386 // Set everything else to 0. 387 let offen = 0; 388 let idxen = 0; 389 let addr64 = 0; 390 let has_vdata = 0; 391 let has_vaddr = 0; 392 let has_glc = 0; 393 let has_dlc = 0; 394 let glc_value = 0; 395 let dlc_value = 0; 396 let has_srsrc = 0; 397 let has_soffset = 0; 398 let has_offset = 0; 399 let has_slc = 0; 400 let has_tfe = 0; 401 let has_sccb = 0; 402 let sccb_value = 0; 403} 404 405class getMUBUFInsDA<list<RegisterClass> vdataList, 406 list<RegisterClass> vaddrList=[], 407 bit isLds = 0> { 408 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 409 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 410 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 411 dag InsNoData = !if(!empty(vaddrList), 412 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 413 offset:$offset, CPol_0:$cpol), 414 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 415 offset:$offset, CPol_0:$cpol) 416 ); 417 dag InsData = !if(!empty(vaddrList), 418 (ins vdata_op:$vdata, SReg_128:$srsrc, 419 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 420 (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 421 SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 422 ); 423 dag ret = !con( 424 !if(!empty(vdataList), InsNoData, InsData), 425 !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz)) 426 ); 427} 428 429class getMUBUFElements<ValueType vt> { 430 int ret = 431 !if(!eq(vt, f16), 1, 432 !if(!eq(vt, v2f16), 2, 433 !if(!eq(vt, v3f16), 3, 434 !if(!eq(vt, v4f16), 4, 435 !if(!eq(vt.Size, 32), 1, 436 !if(!eq(vt.Size, 64), 2, 437 !if(!eq(vt.Size, 96), 3, 438 !if(!eq(vt.Size, 128), 4, 0) 439 ) 440 ) 441 ) 442 ) 443 ) 444 ) 445 ); 446} 447 448class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 449 dag ret = 450 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 451 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 452 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 453 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 454 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 455 (ins)))))); 456} 457 458class getMUBUFAsmOps<int addrKind> { 459 string Pfx = 460 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 461 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 462 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 463 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 464 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 465 ""))))); 466 string ret = Pfx # "$offset"; 467} 468 469class MUBUF_SetupAddr<int addrKind> { 470 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 471 !eq(addrKind, BUFAddrKind.BothEn)); 472 473 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 474 !eq(addrKind, BUFAddrKind.BothEn)); 475 476 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 477 478 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 479} 480 481class MUBUF_Load_Pseudo <string opName, 482 int addrKind, 483 ValueType vdata_vt, 484 bit HasTiedDest = 0, 485 bit isLds = 0, 486 bit isLdsOpc = 0, 487 list<dag> pattern=[], 488 // Workaround bug bz30254 489 int addrKindCopy = addrKind, 490 RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret, 491 RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret> 492 : MUBUF_Pseudo<opName, 493 !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)), 494 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 495 !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))), 496 !if(!or(isLds, isLdsOpc), " ", " $vdata, ") # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" # 497 !if(isLds, " lds", "$tfe") # "$swz", 498 pattern>, 499 MUBUF_SetupAddr<addrKindCopy> { 500 let PseudoInstr = opName # !if(isLds, "_lds", "") # 501 "_" # getAddrName<addrKindCopy>.ret; 502 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 503 504 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 505 let LGKM_CNT = isLds; 506 let has_vdata = !not(isLdsOpc); 507 let mayLoad = 1; 508 let mayStore = isLds; 509 let maybeAtomic = 1; 510 let Uses = !if(!or(isLds, isLdsOpc) , [EXEC, M0], [EXEC]); 511 let has_tfe = !not(isLds); 512 let lds = isLds; 513 let elements = getMUBUFElements<vdata_vt>.ret; 514 let VALU = isLds; 515} 516 517class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 518 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 519 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) 520>; 521 522class MUBUF_Addr64_Load_Pat <Instruction inst, 523 ValueType load_vt = i32, 524 SDPatternOperator ld = null_frag> : Pat < 525 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 526 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) 527>; 528 529multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 530 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 531 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 532} 533 534 535// FIXME: tfe can't be an operand because it requires a separate 536// opcode because it needs an N+1 register class dest register. 537multiclass MUBUF_Pseudo_Loads<string opName, 538 ValueType load_vt = i32, 539 bit TiedDest = 0, 540 bit isLds = 0> { 541 542 defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt); 543 544 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 545 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 546 547 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 548 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 549 550 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 551 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 552 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 553 554 let DisableWQM = 1 in { 555 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 556 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 557 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 558 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 559 } 560} 561 562multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> { 563 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt>; 564 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, 0, 1>; 565} 566 567multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName, 568 ValueType load_vt = i32, 569 bit TiedDest = 0, 570 bit isLds = 0, 571 bit isLdsOpc = 1> { 572 573 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 574 575 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc>; 576 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 577 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 578 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>; 579} 580 581class MUBUF_Store_Pseudo <string opName, 582 int addrKind, 583 ValueType store_vt, 584 list<dag> pattern=[], 585 // Workaround bug bz30254 586 int addrKindCopy = addrKind> 587 : MUBUF_Pseudo<opName, 588 (outs), 589 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 590 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz", 591 pattern>, 592 MUBUF_SetupAddr<addrKindCopy> { 593 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 594 let mayLoad = 0; 595 let mayStore = 1; 596 let maybeAtomic = 1; 597 let elements = getMUBUFElements<store_vt>.ret; 598} 599 600multiclass MUBUF_Pseudo_Stores<string opName, 601 ValueType store_vt = i32, 602 SDPatternOperator st = null_frag> { 603 604 defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt); 605 606 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 607 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 608 i16:$offset))]>, 609 MUBUFAddr64Table<0, NAME>; 610 611 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 612 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 613 i16:$offset))]>, 614 MUBUFAddr64Table<1, NAME>; 615 616 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 617 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 618 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 619 620 let DisableWQM = 1 in { 621 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 622 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 623 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 624 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 625 } 626} 627 628class MUBUF_Pseudo_Store_Lds<string opName> 629 : MUBUF_Pseudo<opName, 630 (outs), 631 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz), 632 " $srsrc, $soffset$offset lds$cpol$swz"> { 633 let LGKM_CNT = 1; 634 let mayLoad = 1; 635 let mayStore = 1; 636 let maybeAtomic = 1; 637 638 let has_vdata = 0; 639 let has_vaddr = 0; 640 let has_tfe = 0; 641 let lds = 1; 642 let VALU = 1; 643 644 let Uses = [EXEC, M0]; 645 let AsmMatchConverter = "cvtMubufLds"; 646} 647 648class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 649 list<RegisterClass> vaddrList=[]> { 650 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 651 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret; 652 dag ret = !if(vdata_in, 653 !if(!empty(vaddrList), 654 (ins vdata_op:$vdata_in, 655 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol), 656 (ins vdata_op:$vdata_in, vaddrClass:$vaddr, 657 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol) 658 ), 659 !if(!empty(vaddrList), 660 (ins vdata_op:$vdata, 661 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol), 662 (ins vdata_op:$vdata, vaddrClass:$vaddr, 663 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol) 664 )); 665} 666 667class getMUBUFAtomicIns<int addrKind, 668 RegisterClass vdataClass, 669 bit vdata_in, 670 // Workaround bug bz30254 671 RegisterClass vdataClassCopy=vdataClass> { 672 dag ret = 673 !if(!eq(addrKind, BUFAddrKind.Offset), 674 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 675 !if(!eq(addrKind, BUFAddrKind.OffEn), 676 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 677 !if(!eq(addrKind, BUFAddrKind.IdxEn), 678 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 679 !if(!eq(addrKind, BUFAddrKind.BothEn), 680 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 681 !if(!eq(addrKind, BUFAddrKind.Addr64), 682 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 683 (ins)))))); 684} 685 686class MUBUF_Atomic_Pseudo<string opName, 687 int addrKind, 688 dag outs, 689 dag ins, 690 string asmOps, 691 list<dag> pattern=[], 692 // Workaround bug bz30254 693 int addrKindCopy = addrKind> 694 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 695 MUBUF_SetupAddr<addrKindCopy> { 696 let mayStore = 1; 697 let mayLoad = 1; 698 let hasPostISelHook = 1; 699 let hasSideEffects = 1; 700 let DisableWQM = 1; 701 let has_glc = 0; 702 let has_dlc = 0; 703 let has_tfe = 0; 704 let has_sccb = 1; 705 let maybeAtomic = 1; 706 let AsmMatchConverter = "cvtMubufAtomic"; 707} 708 709class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 710 RegisterClass vdataClass, 711 list<dag> pattern=[], 712 // Workaround bug bz30254 713 int addrKindCopy = addrKind, 714 RegisterClass vdataClassCopy = vdataClass> 715 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 716 (outs), 717 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 718 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 719 pattern>, 720 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 721 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 722 let glc_value = 0; 723 let dlc_value = 0; 724 let sccb_value = 0; 725 let IsAtomicNoRet = 1; 726} 727 728class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 729 RegisterClass vdataClass, 730 list<dag> pattern=[], 731 // Workaround bug bz30254 732 int addrKindCopy = addrKind, 733 RegisterClass vdataClassCopy = vdataClass, 734 RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret> 735 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 736 (outs vdata_op:$vdata), 737 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 738 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol", 739 pattern>, 740 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 741 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 742 let glc_value = 1; 743 let dlc_value = 0; 744 let sccb_value = 0; 745 let IsAtomicRet = 1; 746 let Constraints = "$vdata = $vdata_in"; 747 let DisableEncoding = "$vdata_in"; 748} 749 750multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 751 RegisterClass vdataClass, 752 ValueType vdataType, 753 bit isFP = isFloatType<vdataType>.ret> { 754 let FPAtomic = isFP in 755 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 756 MUBUFAddr64Table <0, NAME>; 757 758 let FPAtomic = isFP in 759 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 760 MUBUFAddr64Table <1, NAME>; 761 762 let FPAtomic = isFP in 763 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 764 765 let FPAtomic = isFP in 766 767 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 768 769 let FPAtomic = isFP in 770 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 771} 772 773multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 774 RegisterClass vdataClass, 775 ValueType vdataType, 776 SDPatternOperator atomic, 777 bit isFP = isFloatType<vdataType>.ret> { 778 let FPAtomic = isFP in 779 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 780 [(set vdataType:$vdata, 781 (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), 782 vdataType:$vdata_in))]>, 783 MUBUFAddr64Table <0, NAME # "_RTN">; 784 785 let FPAtomic = isFP in 786 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 787 [(set vdataType:$vdata, 788 (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 789 vdataType:$vdata_in))]>, 790 MUBUFAddr64Table <1, NAME # "_RTN">; 791 792 let FPAtomic = isFP in 793 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 794 795 let FPAtomic = isFP in 796 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 797 798 let FPAtomic = isFP in 799 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 800} 801 802multiclass MUBUF_Pseudo_Atomics <string opName, 803 RegisterClass vdataClass, 804 ValueType vdataType, 805 SDPatternOperator atomic = null_frag> : 806 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType>, 807 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 808 809 810//===----------------------------------------------------------------------===// 811// MUBUF Instructions 812//===----------------------------------------------------------------------===// 813 814defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 815 "buffer_load_format_x", f32 816>; 817defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 818 "buffer_load_format_xy", v2f32 819>; 820defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 821 "buffer_load_format_xyz", v3f32 822>; 823defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 824 "buffer_load_format_xyzw", v4f32 825>; 826defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 827 "buffer_store_format_x", f32 828>; 829defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 830 "buffer_store_format_xy", v2f32 831>; 832defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 833 "buffer_store_format_xyz", v3f32 834>; 835defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 836 "buffer_store_format_xyzw", v4f32 837>; 838 839let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 840 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 841 "buffer_load_format_d16_x", i32 842 >; 843 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_xy", v2i32 845 >; 846 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 847 "buffer_load_format_d16_xyz", v3i32 848 >; 849 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 850 "buffer_load_format_d16_xyzw", v4i32 851 >; 852 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 853 "buffer_store_format_d16_x", i32 854 >; 855 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 856 "buffer_store_format_d16_xy", v2i32 857 >; 858 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 859 "buffer_store_format_d16_xyz", v3i32 860 >; 861 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 862 "buffer_store_format_d16_xyzw", v4i32 863 >; 864} // End HasUnpackedD16VMem. 865 866let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 867 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 868 "buffer_load_format_d16_x", f16 869 >; 870 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 871 "buffer_load_format_d16_xy", v2f16 872 >; 873 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 874 "buffer_load_format_d16_xyz", v3f16 875 >; 876 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 877 "buffer_load_format_d16_xyzw", v4f16 878 >; 879 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 880 "buffer_store_format_d16_x", f16 881 >; 882 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 883 "buffer_store_format_d16_xy", v2f16 884 >; 885 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 886 "buffer_store_format_d16_xyz", v3f16 887 >; 888 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 889 "buffer_store_format_d16_xyzw", v4f16 890 >; 891} // End HasPackedD16VMem. 892 893defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 894 "buffer_load_ubyte", i32 895>; 896defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 897 "buffer_load_sbyte", i32 898>; 899defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 900 "buffer_load_ushort", i32 901>; 902defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 903 "buffer_load_sshort", i32 904>; 905defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 906 "buffer_load_dword", i32 907>; 908defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 909 "buffer_load_dwordx2", v2i32 910>; 911defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 912 "buffer_load_dwordx3", v3i32 913>; 914defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 915 "buffer_load_dwordx4", v4i32 916>; 917 918defm BUFFER_LOAD_LDS_B32 : MUBUF_Pseudo_Loads_LDSOpc < 919 "buffer_load_lds_b32", i32 920>; 921defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Pseudo_Loads_LDSOpc < 922 "buffer_load_lds_format_x", f32 923>; 924defm BUFFER_LOAD_LDS_I8 : MUBUF_Pseudo_Loads_LDSOpc < 925 "buffer_load_lds_i8", i32 926>; 927defm BUFFER_LOAD_LDS_I16 : MUBUF_Pseudo_Loads_LDSOpc < 928 "buffer_load_lds_i16", i32 929>; 930defm BUFFER_LOAD_LDS_U8 : MUBUF_Pseudo_Loads_LDSOpc < 931 "buffer_load_lds_u8", i32 932>; 933defm BUFFER_LOAD_LDS_U16 : MUBUF_Pseudo_Loads_LDSOpc < 934 "buffer_load_lds_u16", i32 935>; 936 937defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, atomic_load_8_global>; 938defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, atomic_load_16_global>; 939defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i16, atomic_load_8_global>; 940defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i16, atomic_load_16_global>; 941defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 942defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 943defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 944defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 945defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 946defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 947defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 948defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 949defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 950defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 951 952defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 953 "buffer_store_byte", i32, truncstorei8_global 954>; 955defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 956 "buffer_store_short", i32, truncstorei16_global 957>; 958defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 959 "buffer_store_dword", i32, store_global 960>; 961defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 962 "buffer_store_dwordx2", v2i32, store_global 963>; 964defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 965 "buffer_store_dwordx3", v3i32, store_global 966>; 967defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 968 "buffer_store_dwordx4", v4i32, store_global 969>; 970defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_swap", VGPR_32, i32 972>; 973defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_cmpswap", VReg_64, v2i32 975>; 976defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 977 "buffer_atomic_add", VGPR_32, i32 978>; 979defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 980 "buffer_atomic_sub", VGPR_32, i32 981>; 982defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 983 "buffer_atomic_smin", VGPR_32, i32 984>; 985defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 986 "buffer_atomic_umin", VGPR_32, i32 987>; 988defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 989 "buffer_atomic_smax", VGPR_32, i32 990>; 991defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 992 "buffer_atomic_umax", VGPR_32, i32 993>; 994defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 995 "buffer_atomic_and", VGPR_32, i32 996>; 997defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 998 "buffer_atomic_or", VGPR_32, i32 999>; 1000defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 1001 "buffer_atomic_xor", VGPR_32, i32 1002>; 1003defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 1004 "buffer_atomic_inc", VGPR_32, i32 1005>; 1006defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 1007 "buffer_atomic_dec", VGPR_32, i32 1008>; 1009defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 1010 "buffer_atomic_swap_x2", VReg_64, i64 1011>; 1012defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1013 "buffer_atomic_cmpswap_x2", VReg_128, v2i64 1014>; 1015defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 1016 "buffer_atomic_add_x2", VReg_64, i64 1017>; 1018defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 1019 "buffer_atomic_sub_x2", VReg_64, i64 1020>; 1021defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 1022 "buffer_atomic_smin_x2", VReg_64, i64 1023>; 1024defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 1025 "buffer_atomic_umin_x2", VReg_64, i64 1026>; 1027defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 1028 "buffer_atomic_smax_x2", VReg_64, i64 1029>; 1030defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 1031 "buffer_atomic_umax_x2", VReg_64, i64 1032>; 1033defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 1034 "buffer_atomic_and_x2", VReg_64, i64 1035>; 1036defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1037 "buffer_atomic_or_x2", VReg_64, i64 1038>; 1039defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1040 "buffer_atomic_xor_x2", VReg_64, i64 1041>; 1042defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1043 "buffer_atomic_inc_x2", VReg_64, i64 1044>; 1045defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1046 "buffer_atomic_dec_x2", VReg_64, i64 1047>; 1048 1049let SubtargetPredicate = HasGFX10_BEncoding in 1050defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1051 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1052>; 1053 1054let SubtargetPredicate = isGFX8GFX9 in { 1055def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1056} 1057 1058let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1059/* 1060defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1061defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1062*/ 1063 1064def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1065 int_amdgcn_buffer_wbinvl1_sc>; 1066} 1067 1068let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1069 1070defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1071 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1072>; 1073defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1074 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1075>; 1076defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1077 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1078>; 1079 1080} 1081 1082let SubtargetPredicate = isGFX6GFX7GFX10 in { 1083 1084defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1085 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1086>; 1087defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1088 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1089>; 1090defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1091 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1092>; 1093 1094} 1095 1096let SubtargetPredicate = HasD16LoadStore in { 1097 1098defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1099 "buffer_load_ubyte_d16", i32, 1 1100>; 1101 1102defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1103 "buffer_load_ubyte_d16_hi", i32, 1 1104>; 1105 1106defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1107 "buffer_load_sbyte_d16", i32, 1 1108>; 1109 1110defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1111 "buffer_load_sbyte_d16_hi", i32, 1 1112>; 1113 1114defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1115 "buffer_load_short_d16", i32, 1 1116>; 1117 1118defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1119 "buffer_load_short_d16_hi", i32, 1 1120>; 1121 1122defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1123 "buffer_store_byte_d16_hi", i32 1124>; 1125 1126defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1127 "buffer_store_short_d16_hi", i32 1128>; 1129 1130defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1131 "buffer_load_format_d16_hi_x", i32 1132>; 1133defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1134 "buffer_store_format_d16_hi_x", i32 1135>; 1136 1137} // End HasD16LoadStore 1138 1139def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1140 int_amdgcn_buffer_wbinvl1>; 1141 1142let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1143defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN< 1144 "buffer_atomic_add_f32", VGPR_32, f32 1145>; 1146 1147let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1148defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1149 "buffer_atomic_pk_add_f16", VGPR_32, v2f16 1150>; 1151 1152let OtherPredicates = [HasAtomicFaddRtnInsts] in 1153defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN< 1154 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32 1155>; 1156 1157let OtherPredicates = [isGFX90APlus] in 1158defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN < 1159 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32 1160>; 1161 1162//===----------------------------------------------------------------------===// 1163// MTBUF Instructions 1164//===----------------------------------------------------------------------===// 1165 1166defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1167defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1168defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1169defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1170defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1171defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1172defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1173defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1174 1175let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1176 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1177 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1178 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1179 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1180 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1181 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1182 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1183 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1184} // End HasUnpackedD16VMem. 1185 1186let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1187 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1188 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1189 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1190 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1191 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1192 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1193 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1194 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1195} // End HasPackedD16VMem. 1196 1197let SubtargetPredicate = isGFX7Plus in { 1198 1199//===----------------------------------------------------------------------===// 1200// Instruction definitions for CI and newer. 1201//===----------------------------------------------------------------------===// 1202 1203def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1204 int_amdgcn_buffer_wbinvl1_vol>; 1205 1206} // End let SubtargetPredicate = isGFX7Plus 1207 1208let SubtargetPredicate = isGFX90APlus in { 1209 def BUFFER_WBL2 : MUBUF_Invalidate<"buffer_wbl2"> { 1210 let has_glc = 1; 1211 let has_sccb = 1; 1212 let InOperandList = (ins CPol_0:$cpol); 1213 let AsmOperands = "$cpol"; 1214 } 1215 def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> { 1216 let SubtargetPredicate = isGFX90AOnly; 1217 } 1218 1219 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64>; 1220 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64>; 1221 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64>; 1222} // End SubtargetPredicate = isGFX90APlus 1223 1224def BUFFER_INV : MUBUF_Invalidate<"buffer_inv"> { 1225 let SubtargetPredicate = isGFX940Plus; 1226 let has_glc = 1; 1227 let has_sccb = 1; 1228 let InOperandList = (ins CPol_0:$cpol); 1229 let AsmOperands = "$cpol"; 1230} 1231 1232let SubtargetPredicate = isGFX10Plus in { 1233 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1234 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1235} // End SubtargetPredicate = isGFX10Plus 1236 1237//===----------------------------------------------------------------------===// 1238// MUBUF Patterns 1239//===----------------------------------------------------------------------===// 1240 1241//===----------------------------------------------------------------------===// 1242// buffer_load/store_format patterns 1243//===----------------------------------------------------------------------===// 1244 1245multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1246 string opcode, ValueType memoryVt = vt> { 1247 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>); 1248 1249 def : GCNPat< 1250 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1251 timm:$auxiliary, 0)), 1252 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1253 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1254 >; 1255 1256 def : GCNPat< 1257 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1258 timm:$auxiliary, 0)), 1259 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1260 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1261 >; 1262 1263 def : GCNPat< 1264 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1265 timm:$auxiliary, timm)), 1266 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1267 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1268 >; 1269 1270 def : GCNPat< 1271 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1272 timm:$auxiliary, timm)), 1273 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1274 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1275 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1276 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1277 >; 1278} 1279 1280defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1281defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1282defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1283defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1284defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1285defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1286defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1287defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1288 1289let SubtargetPredicate = HasUnpackedD16VMem in { 1290 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1291 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1292 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1293 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1294 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1295 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1296} // End HasUnpackedD16VMem. 1297 1298let SubtargetPredicate = HasPackedD16VMem in { 1299 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1300 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1301 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1302 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1303 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1304 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1305 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1306 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1307 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1308} // End HasPackedD16VMem. 1309 1310defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1311defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1312defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1313defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1314defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1315defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1316defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1317defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1318defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1319defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1320defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1321defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1322defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1323defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1324defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1325defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1326 1327multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1328 string opcode, ValueType memoryVt = vt> { 1329 defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>); 1330 1331 def : GCNPat< 1332 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1333 timm:$auxiliary, 0), 1334 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1335 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1336 >; 1337 1338 def : GCNPat< 1339 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1340 timm:$auxiliary, 0), 1341 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1342 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1343 >; 1344 1345 def : GCNPat< 1346 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1347 timm:$auxiliary, timm), 1348 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1349 (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1350 >; 1351 1352 def : GCNPat< 1353 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1354 timm:$auxiliary, timm), 1355 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1356 getVregSrcForVT<vt>.ret:$vdata, 1357 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1358 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary), 1359 0, (extract_swz $auxiliary)) 1360 >; 1361} 1362 1363defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1364defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1365defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1366defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1367defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1368defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1369defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1370defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1371 1372let SubtargetPredicate = HasUnpackedD16VMem in { 1373 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1374 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1375 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1376 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1377 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1378 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1379} // End HasUnpackedD16VMem. 1380 1381let SubtargetPredicate = HasPackedD16VMem in { 1382 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1383 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1384 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1385 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1386 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1387 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1388 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1389 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1390 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1391} // End HasPackedD16VMem. 1392 1393defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1394defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1395defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1396defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1397defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1398defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1399defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1400defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1401defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1402defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1403defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1404defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1405defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1406defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1407 1408//===----------------------------------------------------------------------===// 1409// buffer_atomic patterns 1410//===----------------------------------------------------------------------===// 1411 1412multiclass BufferAtomicPat<string OpPrefix, ValueType vt, string Inst, bit isIntr = 0> { 1413 foreach RtnMode = ["ret", "noret"] in { 1414 1415 defvar Op = !cast<SDPatternOperator>(OpPrefix 1416 # !if(!eq(RtnMode, "ret"), "", "_noret") 1417 # !if(isIntr, "", "_" # vt.Size)); 1418 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1419 1420 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1421 def : GCNPat< 1422 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), 1423 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1424 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1425 >; 1426 1427 def : GCNPat< 1428 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1429 vt:$vdata_in)), 1430 (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT<vt>.ret:$vdata_in, 1431 VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) 1432 >; 1433 } // end let AddedComplexity 1434 1435 } // end foreach RtnMode 1436} 1437 1438multiclass BufferAtomicIntrPat<string OpPrefix, ValueType vt, string Inst> { 1439 defm : BufferAtomicPat<OpPrefix, vt, Inst, /* isIntr */ 1>; 1440} 1441 1442multiclass BufferAtomicCmpSwapPat<ValueType vt, ValueType data_vt, string Inst> { 1443 foreach RtnMode = ["ret", "noret"] in { 1444 1445 defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global" 1446 # !if(!eq(RtnMode, "ret"), "", "_noret") 1447 # "_" # vt.Size); 1448 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1449 1450 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1451 defvar OffsetResDag = (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1452 getVregSrcForVT<data_vt>.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, 1453 offset:$offset); 1454 def : GCNPat< 1455 (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), 1456 !if(!eq(RtnMode, "ret"), 1457 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT<data_vt>.ret)), 1458 !if(!eq(vt, i32), sub0, sub0_sub1)), 1459 OffsetResDag) 1460 >; 1461 1462 defvar Addr64ResDag = (!cast<MUBUF_Pseudo>(Inst # "_ADDR64" # InstSuffix) 1463 getVregSrcForVT<data_vt>.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, 1464 SCSrc_b32:$soffset, offset:$offset); 1465 def : GCNPat< 1466 (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), 1467 data_vt:$vdata_in)), 1468 !if(!eq(RtnMode, "ret"), 1469 (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT<data_vt>.ret)), 1470 !if(!eq(vt, i32), sub0, sub0_sub1)), 1471 Addr64ResDag) 1472 >; 1473 } // end let AddedComplexity 1474 1475 } // end foreach RtnMode 1476} 1477 1478foreach Ty = [i32, i64] in { 1479 1480defvar Suffix = !if(!eq(Ty, i64), "_X2", ""); 1481 1482defm : BufferAtomicPat<"atomic_swap_global", Ty, "BUFFER_ATOMIC_SWAP" # Suffix>; 1483defm : BufferAtomicPat<"atomic_load_add_global", Ty, "BUFFER_ATOMIC_ADD" # Suffix>; 1484defm : BufferAtomicPat<"atomic_load_sub_global", Ty, "BUFFER_ATOMIC_SUB" # Suffix>; 1485defm : BufferAtomicPat<"atomic_load_min_global", Ty, "BUFFER_ATOMIC_SMIN" # Suffix>; 1486defm : BufferAtomicPat<"atomic_load_umin_global", Ty, "BUFFER_ATOMIC_UMIN" # Suffix>; 1487defm : BufferAtomicPat<"atomic_load_max_global", Ty, "BUFFER_ATOMIC_SMAX" # Suffix>; 1488defm : BufferAtomicPat<"atomic_load_umax_global", Ty, "BUFFER_ATOMIC_UMAX" # Suffix>; 1489defm : BufferAtomicPat<"atomic_load_and_global", Ty, "BUFFER_ATOMIC_AND" # Suffix>; 1490defm : BufferAtomicPat<"atomic_load_or_global", Ty, "BUFFER_ATOMIC_OR" # Suffix>; 1491defm : BufferAtomicPat<"atomic_load_xor_global", Ty, "BUFFER_ATOMIC_XOR" # Suffix>; 1492defm : BufferAtomicPat<"atomic_inc_global", Ty, "BUFFER_ATOMIC_INC" # Suffix>; 1493defm : BufferAtomicPat<"atomic_dec_global", Ty, "BUFFER_ATOMIC_DEC" # Suffix>; 1494 1495} // end foreach Ty 1496 1497defm : BufferAtomicCmpSwapPat<i32, v2i32, "BUFFER_ATOMIC_CMPSWAP">; 1498defm : BufferAtomicCmpSwapPat<i64, v2i64, "BUFFER_ATOMIC_CMPSWAP_X2">; 1499 1500multiclass SIBufferAtomicPat<string OpPrefix, ValueType vt, string Inst, 1501 list<string> RtnModes = ["ret", "noret"]> { 1502 foreach RtnMode = RtnModes in { 1503 1504 defvar Op = !cast<SDPatternOperator>(OpPrefix 1505 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1506 1507 defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1508 defvar CachePolicy = !if(!eq(RtnMode, "ret"), 1509 (set_glc $cachepolicy), (timm:$cachepolicy)); 1510 1511 let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { 1512 def : GCNPat< 1513 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1514 timm:$offset, timm:$cachepolicy, 0)), 1515 (!cast<MUBUF_Pseudo>(Inst # "_OFFSET" # InstSuffix) 1516 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1517 (as_i16timm $offset), CachePolicy) 1518 >; 1519 1520 def : GCNPat< 1521 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1522 timm:$offset, timm:$cachepolicy, timm)), 1523 (!cast<MUBUF_Pseudo>(Inst # "_IDXEN" # InstSuffix) 1524 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, 1525 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1526 >; 1527 1528 def : GCNPat< 1529 (vt (Op vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1530 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1531 (!cast<MUBUF_Pseudo>(Inst # "_OFFEN" # InstSuffix) 1532 getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, 1533 SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1534 >; 1535 1536 def : GCNPat< 1537 (vt (Op vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1538 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1539 (!cast<MUBUF_Pseudo>(Inst # "_BOTHEN" # InstSuffix) 1540 getVregSrcForVT<vt>.ret:$vdata_in, 1541 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1542 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy) 1543 >; 1544 } // end let AddedComplexity 1545 1546 } // end foreach RtnMode 1547} 1548 1549defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i32, "BUFFER_ATOMIC_SWAP">; 1550defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", f32, "BUFFER_ATOMIC_SWAP">; 1551defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i32, "BUFFER_ATOMIC_ADD">; 1552defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i32, "BUFFER_ATOMIC_SUB">; 1553defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i32, "BUFFER_ATOMIC_SMIN">; 1554defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i32, "BUFFER_ATOMIC_UMIN">; 1555defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i32, "BUFFER_ATOMIC_SMAX">; 1556defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i32, "BUFFER_ATOMIC_UMAX">; 1557defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i32, "BUFFER_ATOMIC_AND">; 1558defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i32, "BUFFER_ATOMIC_OR">; 1559defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i32, "BUFFER_ATOMIC_XOR">; 1560defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i32, "BUFFER_ATOMIC_INC">; 1561defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i32, "BUFFER_ATOMIC_DEC">; 1562defm : SIBufferAtomicPat<"SIbuffer_atomic_csub", i32, "BUFFER_ATOMIC_CSUB", ["ret"]>; 1563defm : SIBufferAtomicPat<"SIbuffer_atomic_swap", i64, "BUFFER_ATOMIC_SWAP_X2">; 1564defm : SIBufferAtomicPat<"SIbuffer_atomic_add", i64, "BUFFER_ATOMIC_ADD_X2">; 1565defm : SIBufferAtomicPat<"SIbuffer_atomic_sub", i64, "BUFFER_ATOMIC_SUB_X2">; 1566defm : SIBufferAtomicPat<"SIbuffer_atomic_smin", i64, "BUFFER_ATOMIC_SMIN_X2">; 1567defm : SIBufferAtomicPat<"SIbuffer_atomic_umin", i64, "BUFFER_ATOMIC_UMIN_X2">; 1568defm : SIBufferAtomicPat<"SIbuffer_atomic_smax", i64, "BUFFER_ATOMIC_SMAX_X2">; 1569defm : SIBufferAtomicPat<"SIbuffer_atomic_umax", i64, "BUFFER_ATOMIC_UMAX_X2">; 1570defm : SIBufferAtomicPat<"SIbuffer_atomic_and", i64, "BUFFER_ATOMIC_AND_X2">; 1571defm : SIBufferAtomicPat<"SIbuffer_atomic_or", i64, "BUFFER_ATOMIC_OR_X2">; 1572defm : SIBufferAtomicPat<"SIbuffer_atomic_xor", i64, "BUFFER_ATOMIC_XOR_X2">; 1573defm : SIBufferAtomicPat<"SIbuffer_atomic_inc", i64, "BUFFER_ATOMIC_INC_X2">; 1574defm : SIBufferAtomicPat<"SIbuffer_atomic_dec", i64, "BUFFER_ATOMIC_DEC_X2">; 1575 1576let SubtargetPredicate = isGFX6GFX7GFX10Plus in { 1577 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f32, "BUFFER_ATOMIC_FMIN">; 1578 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f32, "BUFFER_ATOMIC_FMAX">; 1579} 1580let SubtargetPredicate = isGFX6GFX7GFX10 in { 1581 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_FMIN_X2">; 1582 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_FMAX_X2">; 1583} 1584 1585class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1586 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1587 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1588 [{ return SDValue(N, 0).use_empty(); }]> { 1589 1590 let GISelPredicateCode = [{ 1591 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1592 }]; 1593} 1594 1595multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1596 string opcode> { 1597 def : GCNPat< 1598 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1599 0, i32:$soffset, timm:$offset, 1600 timm:$cachepolicy, 0), 1601 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1602 (as_i16timm $offset), $cachepolicy) 1603 >; 1604 1605 def : GCNPat< 1606 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1607 0, i32:$soffset, timm:$offset, 1608 timm:$cachepolicy, timm), 1609 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1610 (as_i16timm $offset), $cachepolicy) 1611 >; 1612 1613 def : GCNPat< 1614 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1615 i32:$voffset, i32:$soffset, timm:$offset, 1616 timm:$cachepolicy, 0), 1617 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1618 (as_i16timm $offset), $cachepolicy) 1619 >; 1620 1621 def : GCNPat< 1622 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1623 i32:$voffset, i32:$soffset, timm:$offset, 1624 timm:$cachepolicy, timm), 1625 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1626 getVregSrcForVT<vt>.ret:$vdata_in, 1627 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1628 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy) 1629 >; 1630} 1631 1632let SubtargetPredicate = HasAtomicFaddNoRtnInsts in 1633defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1634 1635let SubtargetPredicate = HasAtomicPkFaddNoRtnInsts in 1636defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1637 1638let SubtargetPredicate = HasAtomicFaddRtnInsts in 1639 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f32, "BUFFER_ATOMIC_ADD_F32">; 1640 1641let SubtargetPredicate = isGFX90APlus in { 1642 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1643 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1644 defm : BufferAtomicIntrPat<"int_amdgcn_global_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1645 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1646 1647 defm : SIBufferAtomicPat<"SIbuffer_atomic_fadd", f64, "BUFFER_ATOMIC_ADD_F64">; 1648 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmin", f64, "BUFFER_ATOMIC_MIN_F64">; 1649 defm : SIBufferAtomicPat<"SIbuffer_atomic_fmax", f64, "BUFFER_ATOMIC_MAX_F64">; 1650} // End SubtargetPredicate = isGFX90APlus 1651 1652foreach RtnMode = ["ret", "noret"] in { 1653 1654defvar Op = !cast<SDPatternOperator>(SIbuffer_atomic_cmpswap 1655 # !if(!eq(RtnMode, "ret"), "", "_noret")); 1656defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); 1657defvar CachePolicy = !if(!eq(RtnMode, "ret"), (set_glc $cachepolicy), 1658 (timm:$cachepolicy)); 1659 1660defvar OffsetResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFSET" # InstSuffix) 1661 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1662 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1663def : GCNPat< 1664 (Op 1665 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1666 timm:$offset, timm:$cachepolicy, 0), 1667 !if(!eq(RtnMode, "ret"), 1668 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffsetResDag, VReg_64)), sub0), 1669 OffsetResDag) 1670>; 1671 1672defvar IdxenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_IDXEN" # InstSuffix) 1673 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1674 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1675 CachePolicy); 1676def : GCNPat< 1677 (Op 1678 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1679 0, i32:$soffset, timm:$offset, 1680 timm:$cachepolicy, timm), 1681 !if(!eq(RtnMode, "ret"), 1682 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS IdxenResDag, VReg_64)), sub0), 1683 IdxenResDag) 1684>; 1685 1686defvar OffenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_OFFEN" # InstSuffix) 1687 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1688 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1689 CachePolicy); 1690def : GCNPat< 1691 (Op 1692 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1693 i32:$voffset, i32:$soffset, timm:$offset, 1694 timm:$cachepolicy, 0), 1695 !if(!eq(RtnMode, "ret"), 1696 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS OffenResDag, VReg_64)), sub0), 1697 OffenResDag) 1698>; 1699 1700defvar BothenResDag = (!cast<MUBUF_Pseudo>("BUFFER_ATOMIC_CMPSWAP_BOTHEN" # InstSuffix) 1701 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1702 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1703 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), CachePolicy); 1704def : GCNPat< 1705 (Op 1706 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1707 i32:$voffset, i32:$soffset, timm:$offset, 1708 timm:$cachepolicy, timm), 1709 !if(!eq(RtnMode, "ret"), 1710 (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS BothenResDag, VReg_64)), sub0), 1711 BothenResDag) 1712>; 1713 1714} // end foreach RtnMode 1715 1716class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1717 PatFrag constant_ld> : GCNPat < 1718 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1719 i16:$offset))), 1720 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1721 >; 1722 1723multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1724 ValueType vt, PatFrag atomic_ld> { 1725 def : GCNPat < 1726 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), 1727 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) 1728 >; 1729 1730 def : GCNPat < 1731 (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), 1732 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) 1733 >; 1734} 1735 1736let SubtargetPredicate = isGFX6GFX7 in { 1737def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1738def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1739def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1740def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1741def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1742def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1743 1744defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1745defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1746} // End SubtargetPredicate = isGFX6GFX7 1747 1748multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1749 PatFrag ld> { 1750 1751 def : GCNPat < 1752 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), 1753 (Instr_OFFSET $srsrc, $soffset, $offset) 1754 >; 1755} 1756 1757let OtherPredicates = [Has16BitInsts] in { 1758 1759defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1760defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1761defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1762defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1763defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1764defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1765 1766defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1767 1768} // End OtherPredicates = [Has16BitInsts] 1769 1770multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1771 MUBUF_Pseudo InstrOffset, 1772 ValueType vt, PatFrag ld> { 1773 def : GCNPat < 1774 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1775 i32:$soffset, u16imm:$offset))), 1776 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1777 >; 1778 1779 def : GCNPat < 1780 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1781 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1782 >; 1783} 1784 1785// XXX - Is it possible to have a complex pattern in a PatFrag? 1786multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1787 MUBUF_Pseudo InstrOffset, 1788 ValueType vt, PatFrag ld_frag> { 1789 def : GCNPat < 1790 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1791 (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) 1792 >; 1793 1794 def : GCNPat < 1795 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1796 (InstrOffset $srsrc, $soffset, $offset, $in) 1797 >; 1798} 1799 1800let OtherPredicates = [DisableFlatScratch] in { 1801defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1802defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1803defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1804defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1805defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1806defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1807defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1808defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1809defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1810defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1811 1812foreach vt = Reg32Types.types in { 1813defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1814} 1815defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1816defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1817defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1818 1819let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1820defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1821defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1822defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1823defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1824defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1825defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1826 1827defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1828defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1829defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1830defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1831defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1832defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1833} 1834 1835} // End OtherPredicates = [DisableFlatScratch] 1836 1837multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1838 ValueType vt, PatFrag atomic_st> { 1839 // Store follows atomic op convention so address is first 1840 def : GCNPat < 1841 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), 1842 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) 1843 >; 1844 1845 def : GCNPat < 1846 (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1847 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) 1848 >; 1849} 1850let SubtargetPredicate = isGFX6GFX7 in { 1851defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i32, atomic_store_8_global>; 1852defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_BYTE_ADDR64, BUFFER_STORE_BYTE_OFFSET, i16, atomic_store_8_global>; 1853defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i32, atomic_store_16_global>; 1854defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_SHORT_ADDR64, BUFFER_STORE_SHORT_OFFSET, i16, atomic_store_16_global>; 1855defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_32_global>; 1856defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_64_global>; 1857} // End Predicates = isGFX6GFX7 1858 1859 1860multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1861 PatFrag st> { 1862 1863 def : GCNPat < 1864 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), 1865 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) 1866 >; 1867} 1868 1869defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1870defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1871 1872multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1873 MUBUF_Pseudo InstrOffset, 1874 ValueType vt, PatFrag st, 1875 RegisterClass rc = VGPR_32> { 1876 def : GCNPat < 1877 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1878 i32:$soffset, u16imm:$offset)), 1879 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1880 >; 1881 1882 def : GCNPat < 1883 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1884 u16imm:$offset)), 1885 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0) 1886 >; 1887} 1888 1889let OtherPredicates = [DisableFlatScratch] in { 1890defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1891defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1892defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1893defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1894 1895foreach vt = Reg32Types.types in { 1896defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1897} 1898 1899defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1900defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1901defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1902 1903 1904let OtherPredicates = [HasD16LoadStore, DisableFlatScratch] in { 1905 // Hiding the extract high pattern in the PatFrag seems to not 1906 // automatically increase the complexity. 1907let AddedComplexity = 1 in { 1908defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1909defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1910} 1911} 1912} // End OtherPredicates = [DisableFlatScratch] 1913 1914//===----------------------------------------------------------------------===// 1915// MTBUF Patterns 1916//===----------------------------------------------------------------------===// 1917 1918//===----------------------------------------------------------------------===// 1919// tbuffer_load/store_format patterns 1920//===----------------------------------------------------------------------===// 1921 1922multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1923 string opcode, ValueType memoryVt = vt> { 1924 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>); 1925 1926 def : GCNPat< 1927 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1928 timm:$format, timm:$auxiliary, 0)), 1929 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1930 (as_i8timm $format), 1931 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1932 >; 1933 1934 def : GCNPat< 1935 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1936 timm:$format, timm:$auxiliary, timm)), 1937 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1938 (as_i8timm $format), 1939 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1940 >; 1941 1942 def : GCNPat< 1943 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1944 timm:$format, timm:$auxiliary, 0)), 1945 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1946 (as_i8timm $format), 1947 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1948 >; 1949 1950 def : GCNPat< 1951 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1952 timm:$format, timm:$auxiliary, timm)), 1953 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1954 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1955 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1956 (as_i8timm $format), 1957 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1958 >; 1959} 1960 1961defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1962defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1963defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1964defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1965defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1966defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1967defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1968defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1969 1970let SubtargetPredicate = HasUnpackedD16VMem in { 1971 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1972 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1973 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1974 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1975 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1976} // End HasUnpackedD16VMem. 1977 1978let SubtargetPredicate = HasPackedD16VMem in { 1979 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1980 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1981 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1982 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1983 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1984} // End HasPackedD16VMem. 1985 1986multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1987 string opcode, ValueType memoryVt = vt> { 1988 defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>); 1989 1990 def : GCNPat< 1991 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1992 timm:$format, timm:$auxiliary, 0), 1993 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1994 (as_i16timm $offset), (as_i8timm $format), 1995 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 1996 >; 1997 1998 def : GCNPat< 1999 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 2000 timm:$format, timm:$auxiliary, timm), 2001 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 2002 (as_i16timm $offset), (as_i8timm $format), 2003 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2004 >; 2005 2006 def : GCNPat< 2007 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 2008 timm:$format, timm:$auxiliary, 0), 2009 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 2010 (as_i16timm $offset), (as_i8timm $format), 2011 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2012 >; 2013 2014 def : GCNPat< 2015 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 2016 timm:$offset, timm:$format, timm:$auxiliary, timm), 2017 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 2018 getVregSrcForVT<vt>.ret:$vdata, 2019 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 2020 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 2021 (extract_cpol $auxiliary), 0, (extract_swz $auxiliary)) 2022 >; 2023} 2024 2025defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 2026defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 2027defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 2028defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 2029defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 2030defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 2031defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 2032defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 2033 2034let SubtargetPredicate = HasUnpackedD16VMem in { 2035 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2036 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 2037 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 2038 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 2039 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 2040} // End HasUnpackedD16VMem. 2041 2042let SubtargetPredicate = HasPackedD16VMem in { 2043 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 2044 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 2045 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 2046 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 2047 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 2048} // End HasPackedD16VMem. 2049 2050//===----------------------------------------------------------------------===// 2051// Target-specific instruction encodings. 2052//===----------------------------------------------------------------------===// 2053 2054//===----------------------------------------------------------------------===// 2055// Base ENC_MUBUF for GFX6, GFX7, GFX10, GFX11. 2056//===----------------------------------------------------------------------===// 2057 2058class Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11 <MUBUF_Pseudo ps, int ef, 2059 string real_name = ps.Mnemonic> : 2060 MUBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2061 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2062 let Inst{31-26} = 0x38; 2063 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2064 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2065 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2066 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2067} 2068 2069class MUBUF_Real_gfx11<bits<8> op, MUBUF_Pseudo ps, 2070 string real_name = ps.Mnemonic> : 2071 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2072 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2073 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2074 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2075 let Inst{25-18} = op; 2076 let Inst{53} = !if(ps.has_tfe, tfe, ?); 2077 let Inst{54} = ps.offen; 2078 let Inst{55} = ps.idxen; 2079} 2080 2081class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 2082 Base_MUBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2083 let Inst{12} = ps.offen; 2084 let Inst{13} = ps.idxen; 2085 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2086 let Inst{16} = ps.lds; 2087 let Inst{24-18} = op; 2088 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2089 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2090} 2091 2092class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 2093 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 2094 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2095 let Inst{25} = op{7}; 2096} 2097 2098class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 2099 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 2100 let Inst{15} = ps.addr64; 2101} 2102 2103//===----------------------------------------------------------------------===// 2104// MUBUF - GFX11. 2105//===----------------------------------------------------------------------===// 2106 2107let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2108multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name> { 2109 def _BOTHEN_gfx11 : 2110 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), real_name>, 2111 AtomicNoRet<NAME # "_BOTHEN_gfx11", 0>; 2112 def _IDXEN_gfx11 : 2113 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN"), real_name>, 2114 AtomicNoRet<NAME # "_IDXEN_gfx11", 0>; 2115 def _OFFEN_gfx11 : 2116 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN"), real_name>, 2117 AtomicNoRet<NAME # "_OFFEN_gfx11", 0>; 2118 def _OFFSET_gfx11 : 2119 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET"), real_name>, 2120 AtomicNoRet<NAME # "_OFFSET_gfx11", 0>; 2121} 2122 2123multiclass MUBUF_Real_AllAddr_gfx11_Impl<bits<8> op, MUBUF_Pseudo ps> : 2124 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2125multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op> : 2126 MUBUF_Real_AllAddr_gfx11_Impl<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2127 2128class Pre_gfx11_MUBUF_Name <MUBUF_Pseudo ps, string real_name> : 2129 MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2130multiclass MUBUF_Real_AllAddr_gfx11_Renamed<bits<8> op, string real_name> : 2131 MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2132 def : Pre_gfx11_MUBUF_Name<!cast<MUBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2133} 2134 2135let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2136multiclass MUBUF_Real_Atomics_RTN_gfx11_Renamed<bits<8> op, string real_name> { 2137 def _BOTHEN_RTN_gfx11 : 2138 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN"), real_name>, 2139 AtomicNoRet<NAME # "_BOTHEN_gfx11", 1>; 2140 def _IDXEN_RTN_gfx11 : 2141 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN"), real_name>, 2142 AtomicNoRet<NAME # "_IDXEN_gfx11", 1>; 2143 def _OFFEN_RTN_gfx11 : 2144 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN"), real_name>, 2145 AtomicNoRet<NAME # "_OFFEN_gfx11", 1>; 2146 def _OFFSET_RTN_gfx11 : 2147 MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN"), real_name>, 2148 AtomicNoRet<NAME # "_OFFSET_gfx11", 1>; 2149} 2150 2151multiclass MUBUF_Real_Atomics_RTN_gfx11_impl<bits<8> op, MUBUF_Pseudo ps> : 2152 MUBUF_Real_Atomics_RTN_gfx11_Renamed<op, ps.Mnemonic>; 2153multiclass MUBUF_Real_Atomics_RTN_gfx11<bits<8> op> : 2154 MUBUF_Real_Atomics_RTN_gfx11_impl<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2155 2156multiclass MUBUF_Real_Atomics_gfx11<bits<8> op> : 2157 MUBUF_Real_AllAddr_gfx11<op>, 2158 MUBUF_Real_Atomics_RTN_gfx11<op>; 2159 2160multiclass MUBUF_Real_Atomics_gfx11_Renamed<bits<8> op, string real_name> : 2161 MUBUF_Real_AllAddr_gfx11_Renamed<op, real_name>, 2162 MUBUF_Real_Atomics_RTN_gfx11_Renamed<op, real_name>; 2163 2164let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { 2165def BUFFER_GL0_INV_gfx11 : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>; 2166def BUFFER_GL1_INV_gfx11 : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>; 2167} 2168 2169defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x014, "buffer_load_b32">; 2170defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x015, "buffer_load_b64">; 2171defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x016, "buffer_load_b96">; 2172defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x017, "buffer_load_b128">; 2173defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x020, "buffer_load_d16_b16">; 2174defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x008, "buffer_load_d16_format_x">; 2175defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x009, "buffer_load_d16_format_xy">; 2176defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00a, "buffer_load_d16_format_xyz">; 2177defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00b, "buffer_load_d16_format_xyzw">; 2178defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x023, "buffer_load_d16_hi_b16">; 2179defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x026, "buffer_load_d16_hi_format_x">; 2180defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x022, "buffer_load_d16_hi_i8">; 2181defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x021, "buffer_load_d16_hi_u8">; 2182defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01f, "buffer_load_d16_i8">; 2183defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01e, "buffer_load_d16_u8">; 2184defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x000>; 2185defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x001>; 2186defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x002>; 2187defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x003>; 2188defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x011, "buffer_load_i8">; 2189defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x013, "buffer_load_i16">; 2190defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x010, "buffer_load_u8">; 2191defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x012, "buffer_load_u16">; 2192defm BUFFER_LOAD_LDS_B32 : MUBUF_Real_AllAddr_gfx11<0x031>; 2193defm BUFFER_LOAD_LDS_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x032>; 2194defm BUFFER_LOAD_LDS_I8 : MUBUF_Real_AllAddr_gfx11<0x02e>; 2195defm BUFFER_LOAD_LDS_I16 : MUBUF_Real_AllAddr_gfx11<0x030>; 2196defm BUFFER_LOAD_LDS_U8 : MUBUF_Real_AllAddr_gfx11<0x02d>; 2197defm BUFFER_LOAD_LDS_U16 : MUBUF_Real_AllAddr_gfx11<0x02f>; 2198defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx11_Renamed<0x018, "buffer_store_b8">; 2199defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx11_Renamed<0x019, "buffer_store_b16">; 2200defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx11_Renamed<0x01A, "buffer_store_b32">; 2201defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01B, "buffer_store_b64">; 2202defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01C, "buffer_store_b96">; 2203defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx11_Renamed<0x01D, "buffer_store_b128">; 2204defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x00C, "buffer_store_d16_format_x">; 2205defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx11_Renamed<0x00D, "buffer_store_d16_format_xy">; 2206defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx11_Renamed<0x00E, "buffer_store_d16_format_xyz">; 2207defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00F, "buffer_store_d16_format_xyzw">; 2208defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x024, "buffer_store_d16_hi_b8">; 2209defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx11_Renamed<0x025, "buffer_store_d16_hi_b16">; 2210defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x027, "buffer_store_d16_hi_format_x">; 2211defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx11<0x004>; 2212defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx11<0x005>; 2213defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx11<0x006>; 2214defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx11<0x007>; 2215defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomics_gfx11<0x056>; 2216defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx11_Renamed<0x035, "buffer_atomic_add_u32">; 2217defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x043, "buffer_atomic_add_u64">; 2218defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx11_Renamed<0x03C, "buffer_atomic_and_b32">; 2219defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x049, "buffer_atomic_and_b64">; 2220defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">; 2221defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">; 2222defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">; 2223defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx11_Renamed<0x037, "buffer_atomic_csub_u32">; 2224def : MnemonicAlias<"buffer_atomic_csub", "buffer_atomic_csub_u32">, Requires<[isGFX11Plus]>; 2225defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">; 2226defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">; 2227defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx11_Renamed<0x03F, "buffer_atomic_inc_u32">; 2228defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04C, "buffer_atomic_inc_u64">; 2229defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x052, "buffer_atomic_max_f32">; 2230defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x03A, "buffer_atomic_max_i32">; 2231defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x047, "buffer_atomic_max_i64">; 2232defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx11_Renamed<0x03B, "buffer_atomic_max_u32">; 2233defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x048, "buffer_atomic_max_u64">; 2234defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x051, "buffer_atomic_min_f32">; 2235defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x038, "buffer_atomic_min_i32">; 2236defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x045, "buffer_atomic_min_i64">; 2237defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx11_Renamed<0x039, "buffer_atomic_min_u32">; 2238defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x046, "buffer_atomic_min_u64">; 2239defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx11_Renamed<0x03D, "buffer_atomic_or_b32">; 2240defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04A, "buffer_atomic_or_b64">; 2241defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx11_Renamed<0x036, "buffer_atomic_sub_u32">; 2242defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x044, "buffer_atomic_sub_u64">; 2243defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx11_Renamed<0x033, "buffer_atomic_swap_b32">; 2244defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x041, "buffer_atomic_swap_b64">; 2245defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx11_Renamed<0x03E, "buffer_atomic_xor_b32">; 2246defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx11_Renamed<0x04B, "buffer_atomic_xor_b64">; 2247 2248//===----------------------------------------------------------------------===// 2249// MUBUF - GFX10. 2250//===----------------------------------------------------------------------===// 2251 2252let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2253 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 2254 def _BOTHEN_gfx10 : 2255 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2256 def _IDXEN_gfx10 : 2257 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2258 def _OFFEN_gfx10 : 2259 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2260 def _OFFSET_gfx10 : 2261 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2262 } 2263 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 2264 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2265 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2266 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2267 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2268 2269 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2270 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2271 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2272 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2273 } 2274 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 2275 def _BOTHEN_RTN_gfx10 : 2276 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2277 AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>; 2278 def _IDXEN_RTN_gfx10 : 2279 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2280 AtomicNoRet<NAME # "_IDXEN_gfx10", 1>; 2281 def _OFFEN_RTN_gfx10 : 2282 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2283 AtomicNoRet<NAME # "_OFFEN_gfx10", 1>; 2284 def _OFFSET_RTN_gfx10 : 2285 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2286 AtomicNoRet<NAME # "_OFFSET_gfx10", 1>; 2287 } 2288 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 2289 MUBUF_Real_Atomics_RTN_gfx10<op> { 2290 def _BOTHEN_gfx10 : 2291 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2292 AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>; 2293 def _IDXEN_gfx10 : 2294 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2295 AtomicNoRet<NAME # "_IDXEN_gfx10", 0>; 2296 def _OFFEN_gfx10 : 2297 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2298 AtomicNoRet<NAME # "_OFFEN_gfx10", 0>; 2299 def _OFFSET_gfx10 : 2300 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2301 AtomicNoRet<NAME # "_OFFSET_gfx10", 0>; 2302 } 2303} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2304 2305defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 2306defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 2307defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 2308defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 2309defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 2310defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 2311defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 2312defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 2313// FIXME-GFX10: Add following instructions: 2314//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 2315//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 2316defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 2317defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 2318defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 2319defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 2320defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 2321defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 2322defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 2323defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 2324 2325def BUFFER_GL0_INV_gfx10 : 2326 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 2327def BUFFER_GL1_INV_gfx10 : 2328 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 2329 2330//===----------------------------------------------------------------------===// 2331// MUBUF - GFX6, GFX7, GFX10. 2332//===----------------------------------------------------------------------===// 2333 2334let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 2335 multiclass MUBUF_Real_gfx6<bits<8> op> { 2336 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2337 } 2338} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 2339 2340let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 2341 multiclass MUBUF_Real_gfx7<bits<8> op> { 2342 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 2343 } 2344} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 2345 2346let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2347 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 2348 def _ADDR64_gfx6_gfx7 : 2349 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2350 def _BOTHEN_gfx6_gfx7 : 2351 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2352 def _IDXEN_gfx6_gfx7 : 2353 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2354 def _OFFEN_gfx6_gfx7 : 2355 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2356 def _OFFSET_gfx6_gfx7 : 2357 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2358 } 2359 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 2360 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2361 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 2362 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2363 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2364 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2365 2366 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2367 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>; 2368 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2369 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2370 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2371 } 2372 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> { 2373 def _ADDR64_gfx6_gfx7 : 2374 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2375 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>; 2376 def _BOTHEN_gfx6_gfx7 : 2377 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2378 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>; 2379 def _IDXEN_gfx6_gfx7 : 2380 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2381 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>; 2382 def _OFFEN_gfx6_gfx7 : 2383 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2384 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>; 2385 def _OFFSET_gfx6_gfx7 : 2386 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2387 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>; 2388 2389 def _ADDR64_RTN_gfx6_gfx7 : 2390 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>, 2391 AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>; 2392 def _BOTHEN_RTN_gfx6_gfx7 : 2393 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>, 2394 AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>; 2395 def _IDXEN_RTN_gfx6_gfx7 : 2396 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>, 2397 AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>; 2398 def _OFFEN_RTN_gfx6_gfx7 : 2399 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>, 2400 AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>; 2401 def _OFFSET_RTN_gfx6_gfx7 : 2402 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>, 2403 AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>; 2404 } 2405} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2406 2407multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2408 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2409 2410multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2411 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2412 2413multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2414 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2415 2416// FIXME-GFX6: Following instructions are available only on GFX6. 2417//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2418//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2419 2420defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2421defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2422defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2423defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2424defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2425defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2426defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2427defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2428defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2429defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2430defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2431defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2432defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2433defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2434defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2435defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2436defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2437defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2438defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2439defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2440defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2441defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2442 2443defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2444defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2445defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2446defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2447defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2448defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2449defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2450defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2451defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2452defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2453defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2454defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2455defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2456defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2457defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2458defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2459defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2460defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2461defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2462defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2463defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2464defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2465defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2466defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2467defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2468defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2469defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2470defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2471defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2472// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2473defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2474defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2475defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2476 2477defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2478 2479defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2480defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2481def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2482 2483//===----------------------------------------------------------------------===// 2484// Base ENC_MTBUF for GFX6, GFX7, GFX10, GFX11. 2485//===----------------------------------------------------------------------===// 2486 2487class Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<MTBUF_Pseudo ps, int ef, 2488 string real_name = ps.Mnemonic> : 2489 MTBUF_Real<ps, real_name>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2490 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2491 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2492 let Inst{31-26} = 0x3a; //encoding 2493 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2494 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2495 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2496 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2497} 2498 2499class Base_MTBUF_Real_gfx11<bits<4> op, MTBUF_Pseudo ps, 2500 string real_name = ps.Mnemonic> : 2501 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, SIEncodingFamily.GFX11, real_name> { 2502 let Inst{12} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2503 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2504 let Inst{18-15} = op; 2505 let Inst{25-19} = format; 2506 let Inst{53} = !if(ps.has_tfe, tfe, ?); 2507 let Inst{54} = ps.offen; 2508 let Inst{55} = ps.idxen; 2509} 2510 2511class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2512 Base_MTBUF_Real_gfx6_gfx7_gfx10_gfx11<ps, ef> { 2513 let Inst{12} = ps.offen; 2514 let Inst{13} = ps.idxen; 2515 let Inst{18-16} = op; 2516 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2517 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2518} 2519 2520//===----------------------------------------------------------------------===// 2521// MTBUF - GFX11. 2522//===----------------------------------------------------------------------===// 2523 2524let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in 2525multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> { 2526 def _BOTHEN_gfx11 : 2527 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2528 def _IDXEN_gfx11 : 2529 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN"), real_name>; 2530 def _OFFEN_gfx11 : 2531 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN"), real_name>; 2532 def _OFFSET_gfx11 : 2533 Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>; 2534} 2535 2536multiclass MTBUF_Real_AllAddr_gfx11_Impl<bits<4> op, MTBUF_Pseudo ps> 2537 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>; 2538multiclass MTBUF_Real_AllAddr_gfx11<bits<4> op> 2539 : MTBUF_Real_AllAddr_gfx11_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2540 2541 2542class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name> 2543 : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>; 2544multiclass MTBUF_Real_AllAddr_gfx11_Renamed<bits<4> op, string real_name> 2545 : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> { 2546 def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>; 2547} 2548 2549defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x008, "tbuffer_load_d16_format_x">; 2550defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x009, "tbuffer_load_d16_format_xy">; 2551defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00a, "tbuffer_load_d16_format_xyz">; 2552defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">; 2553defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x000>; 2554defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x001>; 2555defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x002>; 2556defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x003>; 2557defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx11_Renamed<0x00c, "tbuffer_store_d16_format_x">; 2558defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx11_Renamed<0x00d, "tbuffer_store_d16_format_xy">; 2559defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx11_Renamed<0x00e, "tbuffer_store_d16_format_xyz">; 2560defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">; 2561defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx11<0x004>; 2562defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx11<0x005>; 2563defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx11<0x006>; 2564defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx11<0x007>; 2565 2566//===----------------------------------------------------------------------===// 2567// MTBUF - GFX10. 2568//===----------------------------------------------------------------------===// 2569 2570class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2571 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2572 let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value); 2573 let Inst{25-19} = format; 2574 let Inst{53} = op{3}; 2575} 2576 2577let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in { 2578 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2579 def _BOTHEN_gfx10 : 2580 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2581 def _IDXEN_gfx10 : 2582 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2583 def _OFFEN_gfx10 : 2584 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2585 def _OFFSET_gfx10 : 2586 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2587 } 2588} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" 2589 2590defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2591defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2592defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2593defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2594defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2595defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2596defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2597defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2598 2599//===----------------------------------------------------------------------===// 2600// MTBUF - GFX6, GFX7, GFX10. 2601//===----------------------------------------------------------------------===// 2602 2603class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2604 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2605 let Inst{15} = ps.addr64; 2606 let Inst{22-19} = dfmt; 2607 let Inst{25-23} = nfmt; 2608} 2609 2610let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2611 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2612 def _ADDR64_gfx6_gfx7 : 2613 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2614 def _BOTHEN_gfx6_gfx7 : 2615 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2616 def _IDXEN_gfx6_gfx7 : 2617 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2618 def _OFFEN_gfx6_gfx7 : 2619 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2620 def _OFFSET_gfx6_gfx7 : 2621 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2622 } 2623} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2624 2625multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2626 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2627 2628defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2629defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2630defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2631defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2632defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2633defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2634defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2635defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2636 2637//===----------------------------------------------------------------------===// 2638// GFX8, GFX9 (VI). 2639//===----------------------------------------------------------------------===// 2640 2641class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc, 2642 bit has_sccb = ps.has_sccb> : 2643 MUBUF_Real<ps>, 2644 Enc64, 2645 SIMCInstr<ps.PseudoInstr, Enc>, 2646 AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0, 2647 !if(ps.IsAtomicRet, 1, ?))> { 2648 2649 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2650 let Inst{12} = ps.offen; 2651 let Inst{13} = ps.idxen; 2652 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2653 let Inst{15} = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2654 let Inst{16} = ps.lds; 2655 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2656 let Inst{24-18} = op; 2657 let Inst{31-26} = 0x38; //encoding 2658 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2659 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2660 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2661 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2662} 2663 2664class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> : 2665 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> { 2666 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2667 let DecoderNamespace = "GFX8"; 2668 2669 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2670} 2671 2672class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps, 2673 bit has_sccb = ps.has_sccb> : 2674 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> { 2675 let AssemblerPredicate = isGFX90APlus; 2676 let DecoderNamespace = "GFX90A"; 2677 let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""), 2678 !subst("$tfe", "", ps.AsmOperands)); 2679 2680 let Inst{55} = acc; 2681} 2682 2683class MUBUF_Real_gfx940 <bits<7> op, MUBUF_Pseudo ps> : 2684 MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX940> { 2685 let AssemblerPredicate = isGFX940Plus; 2686 let DecoderNamespace = "GFX9"; 2687 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2688 2689 let Inst{55} = acc; 2690} 2691 2692multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> { 2693 def _vi : MUBUF_Real_vi<op, ps>; 2694 2695 foreach _ = BoolToList<!not(ps.FPAtomic)>.ret in 2696 def _gfx90a : MUBUF_Real_gfx90a<op, ps>; 2697 2698 foreach _ = BoolToList<ps.FPAtomic>.ret in { 2699 def _gfx90a : MUBUF_Real_gfx90a<op, ps, 0> { 2700 let SubtargetPredicate = isGFX90AOnly; 2701 let AssemblerPredicate = isGFX90AOnly; 2702 } 2703 def _gfx940 : MUBUF_Real_gfx940<op, ps>; 2704 } 2705} 2706 2707multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2708 defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2709 defm _OFFEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2710 defm _IDXEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2711 defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2712} 2713 2714multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2715 2716 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2717 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2718 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2719 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2720 2721 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2722 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2723 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2724 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2725 2726 def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2727 def _OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2728 def _IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2729 def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2730 2731 def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>; 2732 def _LDS_OFFEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>; 2733 def _LDS_IDXEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>; 2734 def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>; 2735} 2736 2737class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2738 MUBUF_Real<ps>, 2739 Enc64, 2740 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2741 let AssemblerPredicate=HasUnpackedD16VMem; 2742 let DecoderNamespace="GFX80_UNPACKED"; 2743 2744 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2745 let Inst{12} = ps.offen; 2746 let Inst{13} = ps.idxen; 2747 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2748 let Inst{16} = ps.lds; 2749 let Inst{17} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2750 let Inst{24-18} = op; 2751 let Inst{31-26} = 0x38; //encoding 2752 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2753 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2754 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2755 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2756 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2757} 2758 2759multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2760 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2761 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2762 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2763 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2764} 2765 2766multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2767 MUBUF_Real_AllAddr_vi<op> { 2768 defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2769 defm _OFFEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2770 defm _IDXEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2771 defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2772} 2773 2774defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2775defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2776defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2777defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2778defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2779defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2780defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2781defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2782let SubtargetPredicate = HasUnpackedD16VMem in { 2783 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2784 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2785 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2786 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2787 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2788 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2789 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2790 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2791} // End HasUnpackedD16VMem. 2792let SubtargetPredicate = HasPackedD16VMem in { 2793 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2794 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2795 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2796 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2797 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2798 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2799 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2800 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2801} // End HasPackedD16VMem. 2802defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2803defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2804defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2805defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2806defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2807defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_vi <0x15>; 2808defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_vi <0x16>; 2809defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_vi <0x17>; 2810defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2811defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2812defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2813defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2814defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2815defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2816defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2817defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2818 2819defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2820defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2821defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2822defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2823defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2824defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2825 2826defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2827defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2828 2829defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2830defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2831defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2832defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2833defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2834defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2835defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2836defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2837defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2838defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2839defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2840defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2841defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2842 2843defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2844defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2845defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2846defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2847defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2848defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2849defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2850defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2851defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2852defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2853defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2854defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2855defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2856 2857defm BUFFER_STORE_LDS_DWORD : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>; 2858 2859let AssemblerPredicate = isGFX8GFX9 in { 2860def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2861def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2862} // End AssemblerPredicate = isGFX8GFX9 2863 2864let SubtargetPredicate = HasAtomicFaddNoRtnInsts in { 2865 2866defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_Atomic_vi <0x4d>; 2867defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>; 2868 2869} // End SubtargetPredicate = HasAtomicFaddNoRtnInsts 2870 2871let SubtargetPredicate = isGFX90APlus in { 2872 defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>; 2873 defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>; 2874 defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>; 2875} // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus 2876 2877def BUFFER_WBL2_gfx90a : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> { 2878 let AsmString = BUFFER_WBL2.Mnemonic; // drop flags 2879 let AssemblerPredicate = isGFX90AOnly; 2880 let SubtargetPredicate = isGFX90AOnly; 2881} 2882def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>; 2883 2884let SubtargetPredicate = isGFX940Plus in { 2885def BUFFER_WBL2_gfx940 : MUBUF_Real_gfx940<0x28, BUFFER_WBL2>; 2886def BUFFER_INV_gfx940 : MUBUF_Real_gfx940<0x29, BUFFER_INV>; 2887} 2888 2889class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> : 2890 MTBUF_Real<ps>, 2891 Enc64, 2892 SIMCInstr<ps.PseudoInstr, Enc> { 2893 2894 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2895 let Inst{12} = ps.offen; 2896 let Inst{13} = ps.idxen; 2897 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2898 let Inst{18-15} = op; 2899 let Inst{22-19} = dfmt; 2900 let Inst{25-23} = nfmt; 2901 let Inst{31-26} = 0x3a; //encoding 2902 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2903 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2904 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2905 let Inst{53} = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value); 2906 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2907 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2908 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2909} 2910 2911class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2912 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> { 2913 let AssemblerPredicate = isGFX8GFX9NotGFX90A; 2914 let DecoderNamespace = "GFX8"; 2915 2916 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2917} 2918 2919class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> : 2920 MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> { 2921 let AssemblerPredicate = isGFX90APlus; 2922 let DecoderNamespace = "GFX90A"; 2923 let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands); 2924 2925 let Inst{55} = acc; 2926} 2927 2928multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> { 2929 def _vi : MTBUF_Real_vi<op, ps>; 2930 def _gfx90a : MTBUF_Real_gfx90a<op, ps>; 2931} 2932 2933multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2934 defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2935 defm _OFFEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2936 defm _IDXEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2937 defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2938} 2939 2940class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2941 MTBUF_Real<ps>, 2942 Enc64, 2943 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2944 let AssemblerPredicate=HasUnpackedD16VMem; 2945 let DecoderNamespace="GFX80_UNPACKED"; 2946 2947 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2948 let Inst{12} = ps.offen; 2949 let Inst{13} = ps.idxen; 2950 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value); 2951 let Inst{18-15} = op; 2952 let Inst{22-19} = dfmt; 2953 let Inst{25-23} = nfmt; 2954 let Inst{31-26} = 0x3a; //encoding 2955 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2956 let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?); 2957 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2958 let Inst{54} = !if(ps.has_slc, cpol{CPolBit.SLC}, ?); 2959 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2960 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2961} 2962 2963multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2964 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2965 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2966 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2967 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2968} 2969 2970defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2971defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2972defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2973defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2974defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2975defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2976defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2977defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2978let SubtargetPredicate = HasUnpackedD16VMem in { 2979 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2980 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2981 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2982 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2983 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2984 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2985 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2986 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2987} // End HasUnpackedD16VMem. 2988let SubtargetPredicate = HasPackedD16VMem in { 2989 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2990 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2991 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2992 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2993 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2994 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2995 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2996 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2997} // End HasUnpackedD16VMem. 2998 2999def MUBUFInfoTable : GenericTable { 3000 let FilterClass = "MUBUF_Pseudo"; 3001 let CppTypeName = "MUBUFInfo"; 3002 let Fields = [ 3003 "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset", 3004 "IsBufferInv" 3005 ]; 3006 3007 let PrimaryKey = ["Opcode"]; 3008 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 3009} 3010 3011def getMUBUFInfoFromOpcode : SearchIndex { 3012 let Table = MUBUFInfoTable; 3013 let Key = ["Opcode"]; 3014} 3015 3016def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3017 let Table = MUBUFInfoTable; 3018 let Key = ["BaseOpcode", "elements"]; 3019} 3020 3021def MTBUFInfoTable : GenericTable { 3022 let FilterClass = "MTBUF_Pseudo"; 3023 let CppTypeName = "MTBUFInfo"; 3024 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 3025 3026 let PrimaryKey = ["Opcode"]; 3027 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 3028} 3029 3030def getMTBUFInfoFromOpcode : SearchIndex { 3031 let Table = MTBUFInfoTable; 3032 let Key = ["Opcode"]; 3033} 3034 3035def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 3036 let Table = MTBUFInfoTable; 3037 let Key = ["BaseOpcode", "elements"]; 3038} 3039