1//===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUFGetBaseOpcode<string Op> { 58 string ret = !subst("FORMAT_XY", "FORMAT_X", 59 !subst("FORMAT_XYZ", "FORMAT_X", 60 !subst("FORMAT_XYZW", "FORMAT_X", Op))); 61} 62 63class getMTBUFElements<string Op> { 64 int ret = 1; 65} 66 67 68class MTBUF_Pseudo <string opName, dag outs, dag ins, 69 string asmOps, list<dag> pattern=[]> : 70 InstSI<outs, ins, "", pattern>, 71 SIMCInstr<opName, SIEncodingFamily.NONE> { 72 73 let isPseudo = 1; 74 let isCodeGenOnly = 1; 75 let Size = 8; 76 let UseNamedOperandTable = 1; 77 78 string Mnemonic = opName; 79 string AsmOperands = asmOps; 80 81 Instruction Opcode = !cast<Instruction>(NAME); 82 Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret); 83 84 let VM_CNT = 1; 85 let EXP_CNT = 1; 86 let MTBUF = 1; 87 let Uses = [EXEC]; 88 let hasSideEffects = 0; 89 let SchedRW = [WriteVMEM]; 90 91 let AsmMatchConverter = "cvtMtbuf"; 92 93 bits<1> offen = 0; 94 bits<1> idxen = 0; 95 bits<1> addr64 = 0; 96 bits<1> has_vdata = 1; 97 bits<1> has_vaddr = 1; 98 bits<1> has_glc = 1; 99 bits<1> has_dlc = 1; 100 bits<1> glc_value = 0; // the value for glc if no such operand 101 bits<1> dlc_value = 0; // the value for dlc if no such operand 102 bits<1> has_srsrc = 1; 103 bits<1> has_soffset = 1; 104 bits<1> has_offset = 1; 105 bits<1> has_slc = 1; 106 bits<1> has_tfe = 1; 107 bits<4> elements = 0; 108} 109 110class MTBUF_Real <MTBUF_Pseudo ps> : 111 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 112 113 let isPseudo = 0; 114 let isCodeGenOnly = 0; 115 116 // copy relevant pseudo op flags 117 let UseNamedOperandTable = ps.UseNamedOperandTable; 118 let SubtargetPredicate = ps.SubtargetPredicate; 119 let AsmMatchConverter = ps.AsmMatchConverter; 120 let Constraints = ps.Constraints; 121 let DisableEncoding = ps.DisableEncoding; 122 let TSFlags = ps.TSFlags; 123 124 bits<12> offset; 125 bits<1> glc; 126 bits<1> dlc; 127 bits<7> format; 128 bits<8> vaddr; 129 bits<8> vdata; 130 bits<7> srsrc; 131 bits<1> slc; 132 bits<1> tfe; 133 bits<8> soffset; 134 135 bits<4> dfmt = format{3-0}; 136 bits<3> nfmt = format{6-4}; 137} 138 139class getMTBUFInsDA<list<RegisterClass> vdataList, 140 list<RegisterClass> vaddrList=[]> { 141 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 142 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 143 dag InsNoData = !if(!empty(vaddrList), 144 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 145 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 146 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 147 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 148 ); 149 dag InsData = !if(!empty(vaddrList), 150 (ins vdataClass:$vdata, SReg_128:$srsrc, 151 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 152 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz), 153 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 154 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 155 SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz) 156 ); 157 dag ret = !if(!empty(vdataList), InsNoData, InsData); 158} 159 160class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 161 dag ret = 162 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 163 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 (ins)))))); 168} 169 170class getMTBUFAsmOps<int addrKind> { 171 string Pfx = 172 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset", 173 !if(!eq(addrKind, BUFAddrKind.OffEn), 174 "$vaddr, $srsrc,$format $soffset offen", 175 !if(!eq(addrKind, BUFAddrKind.IdxEn), 176 "$vaddr, $srsrc,$format $soffset idxen", 177 !if(!eq(addrKind, BUFAddrKind.BothEn), 178 "$vaddr, $srsrc,$format $soffset idxen offen", 179 !if(!eq(addrKind, BUFAddrKind.Addr64), 180 "$vaddr, $srsrc,$format $soffset addr64", 181 ""))))); 182 string ret = Pfx # "$offset"; 183} 184 185class MTBUF_SetupAddr<int addrKind> { 186 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 187 !eq(addrKind, BUFAddrKind.BothEn)); 188 189 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 190 !eq(addrKind, BUFAddrKind.BothEn)); 191 192 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 193 194 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 195} 196 197class MTBUF_Load_Pseudo <string opName, 198 int addrKind, 199 RegisterClass vdataClass, 200 int elems, 201 list<dag> pattern=[], 202 // Workaround bug bz30254 203 int addrKindCopy = addrKind> 204 : MTBUF_Pseudo<opName, 205 (outs vdataClass:$vdata), 206 getMTBUFIns<addrKindCopy>.ret, 207 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 208 pattern>, 209 MTBUF_SetupAddr<addrKindCopy> { 210 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 211 let mayLoad = 1; 212 let mayStore = 0; 213 let elements = elems; 214} 215 216multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 217 int elems, ValueType load_vt = i32, 218 SDPatternOperator ld = null_frag> { 219 220 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 221 [(set load_vt:$vdata, 222 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 223 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 224 MTBUFAddr64Table<0, NAME>; 225 226 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 227 [(set load_vt:$vdata, 228 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 229 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>, 230 MTBUFAddr64Table<1, NAME>; 231 232 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 233 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 234 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 235 236 let DisableWQM = 1 in { 237 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 238 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 239 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 240 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 241 } 242} 243 244class MTBUF_Store_Pseudo <string opName, 245 int addrKind, 246 RegisterClass vdataClass, 247 int elems, 248 list<dag> pattern=[], 249 // Workaround bug bz30254 250 int addrKindCopy = addrKind, 251 RegisterClass vdataClassCopy = vdataClass> 252 : MTBUF_Pseudo<opName, 253 (outs), 254 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 255 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 256 pattern>, 257 MTBUF_SetupAddr<addrKindCopy> { 258 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 259 let mayLoad = 0; 260 let mayStore = 1; 261 let elements = elems; 262} 263 264multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 265 int elems, ValueType store_vt = i32, 266 SDPatternOperator st = null_frag> { 267 268 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, 269 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 270 i16:$offset, i8:$format, i1:$glc, 271 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 272 MTBUFAddr64Table<0, NAME>; 273 274 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, 275 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 276 i16:$offset, i8:$format, i1:$glc, 277 i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 278 MTBUFAddr64Table<1, NAME>; 279 280 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 281 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 282 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 283 284 let DisableWQM = 1 in { 285 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>; 286 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>; 287 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>; 288 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>; 289 } 290} 291 292 293//===----------------------------------------------------------------------===// 294// MUBUF classes 295//===----------------------------------------------------------------------===// 296 297class MUBUFGetBaseOpcode<string Op> { 298 string ret = !subst("DWORDX2", "DWORD", 299 !subst("DWORDX3", "DWORD", 300 !subst("DWORDX4", "DWORD", Op))); 301} 302 303class MUBUF_Pseudo <string opName, dag outs, dag ins, 304 string asmOps, list<dag> pattern=[]> : 305 InstSI<outs, ins, "", pattern>, 306 SIMCInstr<opName, SIEncodingFamily.NONE> { 307 308 let isPseudo = 1; 309 let isCodeGenOnly = 1; 310 let Size = 8; 311 let UseNamedOperandTable = 1; 312 313 string Mnemonic = opName; 314 string AsmOperands = asmOps; 315 316 Instruction Opcode = !cast<Instruction>(NAME); 317 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 318 319 let VM_CNT = 1; 320 let EXP_CNT = 1; 321 let MUBUF = 1; 322 let Uses = [EXEC]; 323 let hasSideEffects = 0; 324 let SchedRW = [WriteVMEM]; 325 326 let AsmMatchConverter = "cvtMubuf"; 327 328 bits<1> offen = 0; 329 bits<1> idxen = 0; 330 bits<1> addr64 = 0; 331 bits<1> lds = 0; 332 bits<1> has_vdata = 1; 333 bits<1> has_vaddr = 1; 334 bits<1> has_glc = 1; 335 bits<1> has_dlc = 1; 336 bits<1> glc_value = 0; // the value for glc if no such operand 337 bits<1> dlc_value = 0; // the value for dlc if no such operand 338 bits<1> has_srsrc = 1; 339 bits<1> has_soffset = 1; 340 bits<1> has_offset = 1; 341 bits<1> has_slc = 1; 342 bits<1> has_tfe = 1; 343 bits<4> elements = 0; 344} 345 346class MUBUF_Real <MUBUF_Pseudo ps> : 347 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 348 349 let isPseudo = 0; 350 let isCodeGenOnly = 0; 351 352 // copy relevant pseudo op flags 353 let SubtargetPredicate = ps.SubtargetPredicate; 354 let AsmMatchConverter = ps.AsmMatchConverter; 355 let OtherPredicates = ps.OtherPredicates; 356 let Constraints = ps.Constraints; 357 let DisableEncoding = ps.DisableEncoding; 358 let TSFlags = ps.TSFlags; 359 let UseNamedOperandTable = ps.UseNamedOperandTable; 360 361 bits<12> offset; 362 bits<1> glc; 363 bits<1> dlc; 364 bits<8> vaddr; 365 bits<8> vdata; 366 bits<7> srsrc; 367 bits<1> slc; 368 bits<1> tfe; 369 bits<8> soffset; 370} 371 372 373// For cache invalidation instructions. 374class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 375 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 376 377 let AsmMatchConverter = ""; 378 379 let hasSideEffects = 1; 380 let mayLoad = 0; 381 let mayStore = 0; 382 383 // Set everything to 0. 384 let offen = 0; 385 let idxen = 0; 386 let addr64 = 0; 387 let has_vdata = 0; 388 let has_vaddr = 0; 389 let has_glc = 0; 390 let has_dlc = 0; 391 let glc_value = 0; 392 let dlc_value = 0; 393 let has_srsrc = 0; 394 let has_soffset = 0; 395 let has_offset = 0; 396 let has_slc = 0; 397 let has_tfe = 0; 398} 399 400class getMUBUFInsDA<list<RegisterClass> vdataList, 401 list<RegisterClass> vaddrList=[], 402 bit isLds = 0> { 403 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 404 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 405 dag InsNoData = !if(!empty(vaddrList), 406 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 407 offset:$offset, GLC:$glc, SLC:$slc), 408 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 409 offset:$offset, GLC:$glc, SLC:$slc) 410 ); 411 dag InsData = !if(!empty(vaddrList), 412 (ins vdataClass:$vdata, SReg_128:$srsrc, 413 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 414 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 415 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 416 ); 417 dag ret = !con( 418 !if(!empty(vdataList), InsNoData, InsData), 419 !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz)) 420 ); 421} 422 423class getMUBUFElements<ValueType vt> { 424 // eq does not support ValueType for some reason. 425 string vtAsStr = !cast<string>(vt); 426 427 int ret = 428 !if(!eq(vtAsStr, "f16"), 1, 429 !if(!eq(vtAsStr, "v2f16"), 2, 430 !if(!eq(vtAsStr, "v3f16"), 3, 431 !if(!eq(vtAsStr, "v4f16"), 4, 432 !if(!eq(vt.Size, 32), 1, 433 !if(!eq(vt.Size, 64), 2, 434 !if(!eq(vt.Size, 96), 3, 435 !if(!eq(vt.Size, 128), 4, 0) 436 ) 437 ) 438 ) 439 ) 440 ) 441 ) 442 ); 443} 444 445class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 446 dag ret = 447 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 448 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 449 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 450 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 451 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 452 (ins)))))); 453} 454 455class getMUBUFAsmOps<int addrKind> { 456 string Pfx = 457 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 458 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 459 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 460 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 461 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 462 ""))))); 463 string ret = Pfx # "$offset"; 464} 465 466class MUBUF_SetupAddr<int addrKind> { 467 bits<1> offen = !or(!eq(addrKind, BUFAddrKind.OffEn), 468 !eq(addrKind, BUFAddrKind.BothEn)); 469 470 bits<1> idxen = !or(!eq(addrKind, BUFAddrKind.IdxEn), 471 !eq(addrKind, BUFAddrKind.BothEn)); 472 473 bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64); 474 475 bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset); 476} 477 478class MUBUF_Load_Pseudo <string opName, 479 int addrKind, 480 ValueType vdata_vt, 481 bit HasTiedDest = 0, 482 bit isLds = 0, 483 list<dag> pattern=[], 484 // Workaround bug bz30254 485 int addrKindCopy = addrKind> 486 : MUBUF_Pseudo<opName, 487 (outs getVregSrcForVT<vdata_vt>.ret:$vdata), 488 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 489 !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))), 490 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 491 !if(isLds, " lds", "$tfe") # "$dlc$swz", 492 pattern>, 493 MUBUF_SetupAddr<addrKindCopy> { 494 let PseudoInstr = opName # !if(isLds, "_lds", "") # 495 "_" # getAddrName<addrKindCopy>.ret; 496 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 497 498 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 499 let mayLoad = 1; 500 let mayStore = 0; 501 let maybeAtomic = 1; 502 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 503 let has_tfe = !not(isLds); 504 let lds = isLds; 505 let elements = getMUBUFElements<vdata_vt>.ret; 506} 507 508class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 509 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 510 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 511>; 512 513class MUBUF_Addr64_Load_Pat <Instruction inst, 514 ValueType load_vt = i32, 515 SDPatternOperator ld = null_frag> : Pat < 516 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 517 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)) 518>; 519 520multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 521 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 522 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 523} 524 525 526// FIXME: tfe can't be an operand because it requires a separate 527// opcode because it needs an N+1 register class dest register. 528multiclass MUBUF_Pseudo_Loads<string opName, 529 ValueType load_vt = i32, 530 SDPatternOperator ld = null_frag, 531 bit TiedDest = 0, 532 bit isLds = 0> { 533 534 defvar legal_load_vt = !if(!eq(!cast<string>(load_vt), !cast<string>(v3f16)), v4f16, load_vt); 535 536 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>, 537 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 538 539 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>, 540 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 541 542 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 543 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 544 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 545 546 let DisableWQM = 1 in { 547 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>; 548 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>; 549 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>; 550 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>; 551 } 552} 553 554multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32, 555 SDPatternOperator ld_nolds = null_frag, 556 SDPatternOperator ld_lds = null_frag> { 557 defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>; 558 defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>; 559} 560 561class MUBUF_Store_Pseudo <string opName, 562 int addrKind, 563 ValueType store_vt, 564 list<dag> pattern=[], 565 // Workaround bug bz30254 566 int addrKindCopy = addrKind> 567 : MUBUF_Pseudo<opName, 568 (outs), 569 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret, 570 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz", 571 pattern>, 572 MUBUF_SetupAddr<addrKindCopy> { 573 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 574 let mayLoad = 0; 575 let mayStore = 1; 576 let maybeAtomic = 1; 577 let elements = getMUBUFElements<store_vt>.ret; 578} 579 580multiclass MUBUF_Pseudo_Stores<string opName, 581 ValueType store_vt = i32, 582 SDPatternOperator st = null_frag> { 583 584 defvar legal_store_vt = !if(!eq(!cast<string>(store_vt), !cast<string>(v3f16)), v4f16, store_vt); 585 586 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, 587 [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 588 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 589 MUBUFAddr64Table<0, NAME>; 590 591 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, 592 [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 593 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>, 594 MUBUFAddr64Table<1, NAME>; 595 596 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 597 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 598 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 599 600 let DisableWQM = 1 in { 601 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>; 602 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>; 603 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>; 604 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>; 605 } 606} 607 608class MUBUF_Pseudo_Store_Lds<string opName> 609 : MUBUF_Pseudo<opName, 610 (outs), 611 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz), 612 " $srsrc, $soffset$offset lds$glc$slc$swz"> { 613 let mayLoad = 0; 614 let mayStore = 1; 615 let maybeAtomic = 1; 616 617 let has_vdata = 0; 618 let has_vaddr = 0; 619 let has_tfe = 0; 620 let lds = 1; 621 622 let Uses = [EXEC, M0]; 623 let AsmMatchConverter = "cvtMubufLds"; 624} 625 626class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 627 list<RegisterClass> vaddrList=[]> { 628 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 629 dag ret = !if(vdata_in, 630 !if(!empty(vaddrList), 631 (ins vdataClass:$vdata_in, 632 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC_1:$glc1, SLC:$slc), 633 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 634 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC_1:$glc1, SLC:$slc) 635 ), 636 !if(!empty(vaddrList), 637 (ins vdataClass:$vdata, 638 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 639 (ins vdataClass:$vdata, vaddrClass:$vaddr, 640 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 641 )); 642} 643 644class getMUBUFAtomicIns<int addrKind, 645 RegisterClass vdataClass, 646 bit vdata_in, 647 // Workaround bug bz30254 648 RegisterClass vdataClassCopy=vdataClass> { 649 dag ret = 650 !if(!eq(addrKind, BUFAddrKind.Offset), 651 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 652 !if(!eq(addrKind, BUFAddrKind.OffEn), 653 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 654 !if(!eq(addrKind, BUFAddrKind.IdxEn), 655 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 656 !if(!eq(addrKind, BUFAddrKind.BothEn), 657 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 658 !if(!eq(addrKind, BUFAddrKind.Addr64), 659 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 660 (ins)))))); 661} 662 663class MUBUF_Atomic_Pseudo<string opName, 664 int addrKind, 665 dag outs, 666 dag ins, 667 string asmOps, 668 list<dag> pattern=[], 669 // Workaround bug bz30254 670 int addrKindCopy = addrKind> 671 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 672 MUBUF_SetupAddr<addrKindCopy> { 673 let mayStore = 1; 674 let mayLoad = 1; 675 let hasPostISelHook = 1; 676 let hasSideEffects = 1; 677 let DisableWQM = 1; 678 let has_glc = 0; 679 let has_dlc = 0; 680 let has_tfe = 0; 681 let maybeAtomic = 1; 682} 683 684class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 685 RegisterClass vdataClass, 686 list<dag> pattern=[], 687 // Workaround bug bz30254 688 int addrKindCopy = addrKind, 689 RegisterClass vdataClassCopy = vdataClass> 690 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 691 (outs), 692 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 693 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 694 pattern>, 695 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 696 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 697 let glc_value = 0; 698 let dlc_value = 0; 699 let AsmMatchConverter = "cvtMubufAtomic"; 700} 701 702class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 703 RegisterClass vdataClass, 704 list<dag> pattern=[], 705 // Workaround bug bz30254 706 int addrKindCopy = addrKind, 707 RegisterClass vdataClassCopy = vdataClass> 708 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 709 (outs vdataClassCopy:$vdata), 710 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 711 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc1$slc", 712 pattern>, 713 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 714 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 715 let glc_value = 1; 716 let dlc_value = 0; 717 let Constraints = "$vdata = $vdata_in"; 718 let DisableEncoding = "$vdata_in"; 719 let AsmMatchConverter = "cvtMubufAtomicReturn"; 720} 721 722multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 723 RegisterClass vdataClass, 724 ValueType vdataType, 725 SDPatternOperator atomic, 726 bit isFP = isFloatType<vdataType>.ret> { 727 let FPAtomic = isFP in 728 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 729 MUBUFAddr64Table <0, NAME>; 730 731 let FPAtomic = isFP in 732 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 733 MUBUFAddr64Table <1, NAME>; 734 735 let FPAtomic = isFP in 736 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 737 738 let FPAtomic = isFP in 739 740 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 741 742 let FPAtomic = isFP in 743 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 744} 745 746multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 747 RegisterClass vdataClass, 748 ValueType vdataType, 749 SDPatternOperator atomic, 750 bit isFP = isFloatType<vdataType>.ret> { 751 let FPAtomic = isFP in 752 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 753 [(set vdataType:$vdata, 754 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 755 vdataType:$vdata_in))]>, 756 MUBUFAddr64Table <0, NAME # "_RTN">; 757 758 let FPAtomic = isFP in 759 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 760 [(set vdataType:$vdata, 761 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 762 vdataType:$vdata_in))]>, 763 MUBUFAddr64Table <1, NAME # "_RTN">; 764 765 let FPAtomic = isFP in 766 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 767 768 let FPAtomic = isFP in 769 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 770 771 let FPAtomic = isFP in 772 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 773} 774 775multiclass MUBUF_Pseudo_Atomics <string opName, 776 RegisterClass vdataClass, 777 ValueType vdataType, 778 SDPatternOperator atomic> : 779 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 780 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 781 782 783//===----------------------------------------------------------------------===// 784// MUBUF Instructions 785//===----------------------------------------------------------------------===// 786 787defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 788 "buffer_load_format_x", f32 789>; 790defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 791 "buffer_load_format_xy", v2f32 792>; 793defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 794 "buffer_load_format_xyz", v3f32 795>; 796defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 797 "buffer_load_format_xyzw", v4f32 798>; 799defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 800 "buffer_store_format_x", f32 801>; 802defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 803 "buffer_store_format_xy", v2f32 804>; 805defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 806 "buffer_store_format_xyz", v3f32 807>; 808defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 809 "buffer_store_format_xyzw", v4f32 810>; 811 812let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 813 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 814 "buffer_load_format_d16_x", i32 815 >; 816 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 817 "buffer_load_format_d16_xy", v2i32 818 >; 819 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 820 "buffer_load_format_d16_xyz", v3i32 821 >; 822 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 823 "buffer_load_format_d16_xyzw", v4i32 824 >; 825 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 826 "buffer_store_format_d16_x", i32 827 >; 828 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 829 "buffer_store_format_d16_xy", v2i32 830 >; 831 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 832 "buffer_store_format_d16_xyz", v3i32 833 >; 834 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 835 "buffer_store_format_d16_xyzw", v4i32 836 >; 837} // End HasUnpackedD16VMem. 838 839let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 840 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 841 "buffer_load_format_d16_x", f16 842 >; 843 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 844 "buffer_load_format_d16_xy", v2f16 845 >; 846 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 847 "buffer_load_format_d16_xyz", v3f16 848 >; 849 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 850 "buffer_load_format_d16_xyzw", v4f16 851 >; 852 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 853 "buffer_store_format_d16_x", f16 854 >; 855 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 856 "buffer_store_format_d16_xy", v2f16 857 >; 858 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 859 "buffer_store_format_d16_xyz", v3f16 860 >; 861 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 862 "buffer_store_format_d16_xyzw", v4f16 863 >; 864} // End HasPackedD16VMem. 865 866defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 867 "buffer_load_ubyte", i32 868>; 869defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 870 "buffer_load_sbyte", i32 871>; 872defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 873 "buffer_load_ushort", i32 874>; 875defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 876 "buffer_load_sshort", i32 877>; 878defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 879 "buffer_load_dword", i32 880>; 881defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 882 "buffer_load_dwordx2", v2i32 883>; 884defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 885 "buffer_load_dwordx3", v3i32 886>; 887defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 888 "buffer_load_dwordx4", v4i32 889>; 890 891defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 892defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 893defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 894defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 895defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 896defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 897defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 898defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 899defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 900defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 901 902// This is not described in AMD documentation, 903// but 'lds' versions of these opcodes are available 904// in at least GFX8+ chips. See Bug 37653. 905let SubtargetPredicate = isGFX8GFX9 in { 906defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 907 "buffer_load_dwordx2", v2i32, null_frag, 0, 1 908>; 909defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 910 "buffer_load_dwordx3", v3i32, null_frag, 0, 1 911>; 912defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 913 "buffer_load_dwordx4", v4i32, null_frag, 0, 1 914>; 915} 916 917defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 918 "buffer_store_byte", i32, truncstorei8_global 919>; 920defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 921 "buffer_store_short", i32, truncstorei16_global 922>; 923defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 924 "buffer_store_dword", i32, store_global 925>; 926defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 927 "buffer_store_dwordx2", v2i32, store_global 928>; 929defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 930 "buffer_store_dwordx3", v3i32, store_global 931>; 932defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 933 "buffer_store_dwordx4", v4i32, store_global 934>; 935defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 936 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32 937>; 938defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 939 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 940>; 941defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 942 "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32 943>; 944defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 945 "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32 946>; 947defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 948 "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32 949>; 950defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 951 "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32 952>; 953defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 954 "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32 955>; 956defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 957 "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32 958>; 959defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 960 "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32 961>; 962defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 963 "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32 964>; 965defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 966 "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32 967>; 968defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 969 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32 970>; 971defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 972 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32 973>; 974defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 975 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64 976>; 977defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 978 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 979>; 980defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 981 "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64 982>; 983defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 984 "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64 985>; 986defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 987 "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64 988>; 989defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 990 "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64 991>; 992defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 993 "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64 994>; 995defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 996 "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64 997>; 998defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 999 "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64 1000>; 1001defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 1002 "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64 1003>; 1004defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 1005 "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64 1006>; 1007defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 1008 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64 1009>; 1010defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 1011 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64 1012>; 1013 1014let SubtargetPredicate = HasGFX10_BEncoding in 1015defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN < 1016 "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub 1017>; 1018 1019let SubtargetPredicate = isGFX8GFX9 in { 1020def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 1021} 1022 1023let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 1024/* 1025defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 1026defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 1027*/ 1028 1029def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 1030 int_amdgcn_buffer_wbinvl1_sc>; 1031} 1032 1033let SubtargetPredicate = isGFX6GFX7GFX10 in { 1034 1035defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics < 1036 "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag 1037>; 1038defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics < 1039 "buffer_atomic_fmin", VGPR_32, f32, null_frag 1040>; 1041defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics < 1042 "buffer_atomic_fmax", VGPR_32, f32, null_frag 1043>; 1044defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics < 1045 "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag 1046>; 1047defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics < 1048 "buffer_atomic_fmin_x2", VReg_64, f64, null_frag 1049>; 1050defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics < 1051 "buffer_atomic_fmax_x2", VReg_64, f64, null_frag 1052>; 1053 1054} 1055 1056let SubtargetPredicate = HasD16LoadStore in { 1057 1058defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1059 "buffer_load_ubyte_d16", i32, null_frag, 1 1060>; 1061 1062defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1063 "buffer_load_ubyte_d16_hi", i32, null_frag, 1 1064>; 1065 1066defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1067 "buffer_load_sbyte_d16", i32, null_frag, 1 1068>; 1069 1070defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1071 "buffer_load_sbyte_d16_hi", i32, null_frag, 1 1072>; 1073 1074defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1075 "buffer_load_short_d16", i32, null_frag, 1 1076>; 1077 1078defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1079 "buffer_load_short_d16_hi", i32, null_frag, 1 1080>; 1081 1082defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1083 "buffer_store_byte_d16_hi", i32 1084>; 1085 1086defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1087 "buffer_store_short_d16_hi", i32 1088>; 1089 1090defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1091 "buffer_load_format_d16_hi_x", i32 1092>; 1093defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1094 "buffer_store_format_d16_hi_x", i32 1095>; 1096 1097} // End HasD16LoadStore 1098 1099def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1100 int_amdgcn_buffer_wbinvl1>; 1101 1102let SubtargetPredicate = HasAtomicFaddInsts in { 1103defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1104 "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_noret_32 1105>; 1106defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1107 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_noret_32 1108>; 1109} // End SubtargetPredicate = HasAtomicFaddInsts 1110 1111//===----------------------------------------------------------------------===// 1112// MTBUF Instructions 1113//===----------------------------------------------------------------------===// 1114 1115defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32, 1>; 1116defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64, 2>; 1117defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96, 3>; 1118defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128, 4>; 1119defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32, 1>; 1120defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64, 2>; 1121defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96, 3>; 1122defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>; 1123 1124let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1125 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1126 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64, 2>; 1127 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96, 3>; 1128 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128, 4>; 1129 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1130 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64, 2>; 1131 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96, 3>; 1132 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>; 1133} // End HasUnpackedD16VMem. 1134 1135let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1136 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32, 1>; 1137 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32, 2>; 1138 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64, 3>; 1139 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64, 4>; 1140 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32, 1>; 1141 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32, 2>; 1142 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64, 3>; 1143 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>; 1144} // End HasPackedD16VMem. 1145 1146let SubtargetPredicate = isGFX7Plus in { 1147 1148//===----------------------------------------------------------------------===// 1149// Instruction definitions for CI and newer. 1150//===----------------------------------------------------------------------===// 1151 1152def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1153 int_amdgcn_buffer_wbinvl1_vol>; 1154 1155} // End let SubtargetPredicate = isGFX7Plus 1156 1157let SubtargetPredicate = isGFX10Plus in { 1158 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1159 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1160} // End SubtargetPredicate = isGFX10Plus 1161 1162//===----------------------------------------------------------------------===// 1163// MUBUF Patterns 1164//===----------------------------------------------------------------------===// 1165 1166//===----------------------------------------------------------------------===// 1167// buffer_load/store_format patterns 1168//===----------------------------------------------------------------------===// 1169 1170multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1171 string opcode, ValueType memoryVt = vt> { 1172 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mubuf_intrinsic_load<name, memoryVt>); 1173 1174 def : GCNPat< 1175 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1176 timm:$auxiliary, 0)), 1177 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1178 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1179 (extract_swz $auxiliary)) 1180 >; 1181 1182 def : GCNPat< 1183 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1184 timm:$auxiliary, 0)), 1185 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1186 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1187 (extract_swz $auxiliary)) 1188 >; 1189 1190 def : GCNPat< 1191 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1192 timm:$auxiliary, timm)), 1193 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1194 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1195 (extract_swz $auxiliary)) 1196 >; 1197 1198 def : GCNPat< 1199 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1200 timm:$auxiliary, timm)), 1201 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1202 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1203 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1204 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1205 (extract_swz $auxiliary)) 1206 >; 1207} 1208 1209defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1210defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1211defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1212defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1213defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1214defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1215defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1216defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1217 1218let SubtargetPredicate = HasUnpackedD16VMem in { 1219 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1220 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1221 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1222 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1223 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1224 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1225} // End HasUnpackedD16VMem. 1226 1227let SubtargetPredicate = HasPackedD16VMem in { 1228 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1229 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1230 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">; 1231 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1232 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1233 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1234 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>; 1235 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1236 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1237} // End HasPackedD16VMem. 1238 1239defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1240defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1241defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">; 1242defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">; 1243defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1244defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1245defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">; 1246defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">; 1247defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1248defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1249defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1250defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1251defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1252defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1253defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1254defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1255 1256multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1257 string opcode, ValueType memoryVt = vt> { 1258 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mubuf_intrinsic_store<name, memoryVt>); 1259 1260 def : GCNPat< 1261 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1262 timm:$auxiliary, 0), 1263 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1264 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1265 (extract_swz $auxiliary)) 1266 >; 1267 1268 def : GCNPat< 1269 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1270 timm:$auxiliary, 0), 1271 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1272 (as_i16timm $offset), (extract_glc $auxiliary), 1273 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1274 (extract_swz $auxiliary)) 1275 >; 1276 1277 def : GCNPat< 1278 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1279 timm:$auxiliary, timm), 1280 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1281 (as_i16timm $offset), (extract_glc $auxiliary), 1282 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1283 (extract_swz $auxiliary)) 1284 >; 1285 1286 def : GCNPat< 1287 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1288 timm:$auxiliary, timm), 1289 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1290 getVregSrcForVT<vt>.ret:$vdata, 1291 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1292 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_glc $auxiliary), 1293 (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1294 (extract_swz $auxiliary)) 1295 >; 1296} 1297 1298defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1299defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1300defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1301defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1302defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1303defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1304defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1305defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1306 1307let SubtargetPredicate = HasUnpackedD16VMem in { 1308 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1309 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1310 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1311 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1312 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1313 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1314} // End HasUnpackedD16VMem. 1315 1316let SubtargetPredicate = HasPackedD16VMem in { 1317 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1318 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1319 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">; 1320 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1321 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1322 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1323 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>; 1324 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1325 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1326} // End HasPackedD16VMem. 1327 1328defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1329defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1330defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">; 1331defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">; 1332defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1333defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1334defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">; 1335defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">; 1336defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1337defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1338defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1339defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1340defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1341defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1342 1343//===----------------------------------------------------------------------===// 1344// buffer_atomic patterns 1345//===----------------------------------------------------------------------===// 1346 1347multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1348 string opcode> { 1349 def : GCNPat< 1350 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset, 1351 timm:$offset, timm:$cachepolicy, 0)), 1352 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) 1353 getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1354 (as_i16timm $offset), (extract_slc $cachepolicy)) 1355 >; 1356 1357 def : GCNPat< 1358 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, 1359 timm:$offset, timm:$cachepolicy, timm)), 1360 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1361 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1362 (as_i16timm $offset), (extract_slc $cachepolicy)) 1363 >; 1364 1365 def : GCNPat< 1366 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset, 1367 i32:$soffset, timm:$offset, timm:$cachepolicy, 0)), 1368 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in, 1369 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1370 (as_i16timm $offset), (extract_slc $cachepolicy)) 1371 >; 1372 1373 def : GCNPat< 1374 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset, 1375 i32:$soffset, timm:$offset, timm:$cachepolicy, timm)), 1376 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1377 getVregSrcForVT<vt>.ret:$vdata_in, 1378 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1379 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1380 (extract_slc $cachepolicy)) 1381 >; 1382} 1383 1384defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1385defm : BufferAtomicPatterns<SIbuffer_atomic_swap, f32, "BUFFER_ATOMIC_SWAP">; 1386defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1387defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1388defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1389defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1390defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1391defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1392defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1393defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1394defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1395defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">; 1396defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">; 1397defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">; 1398defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1399defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1400defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1401defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1402defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1403defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1404defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1405defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1406defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1407defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1408defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">; 1409defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">; 1410 1411class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag < 1412 (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7), 1413 (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)), 1414 [{ return SDValue(N, 0).use_empty(); }]> { 1415 1416 let GISelPredicateCode = [{ 1417 return MRI.use_nodbg_empty(MI.getOperand(0).getReg()); 1418 }]; 1419} 1420 1421multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1422 string opcode> { 1423 def : GCNPat< 1424 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1425 0, i32:$soffset, timm:$offset, 1426 timm:$cachepolicy, 0), 1427 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset, 1428 (as_i16timm $offset), (extract_slc $cachepolicy)) 1429 >; 1430 1431 def : GCNPat< 1432 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1433 0, i32:$soffset, timm:$offset, 1434 timm:$cachepolicy, timm), 1435 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1436 (as_i16timm $offset), (extract_slc $cachepolicy)) 1437 >; 1438 1439 def : GCNPat< 1440 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0, 1441 i32:$voffset, i32:$soffset, timm:$offset, 1442 timm:$cachepolicy, 0), 1443 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1444 (as_i16timm $offset), (extract_slc $cachepolicy)) 1445 >; 1446 1447 def : GCNPat< 1448 (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1449 i32:$voffset, i32:$soffset, timm:$offset, 1450 timm:$cachepolicy, timm), 1451 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1452 getVregSrcForVT<vt>.ret:$vdata_in, 1453 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1454 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)) 1455 >; 1456} 1457 1458let SubtargetPredicate = HasAtomicFaddInsts in { 1459defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1460defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1461} 1462 1463def : GCNPat< 1464 (SIbuffer_atomic_cmpswap 1465 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset, 1466 timm:$offset, timm:$cachepolicy, 0), 1467 (EXTRACT_SUBREG 1468 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1469 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1470 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1471 (extract_slc $cachepolicy)), sub0) 1472>; 1473 1474def : GCNPat< 1475 (SIbuffer_atomic_cmpswap 1476 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1477 0, i32:$soffset, timm:$offset, 1478 timm:$cachepolicy, timm), 1479 (EXTRACT_SUBREG 1480 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1481 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1482 VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1483 sub0) 1484>; 1485 1486def : GCNPat< 1487 (SIbuffer_atomic_cmpswap 1488 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1489 i32:$voffset, i32:$soffset, timm:$offset, 1490 timm:$cachepolicy, 0), 1491 (EXTRACT_SUBREG 1492 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1493 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1494 VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1495 sub0) 1496>; 1497 1498def : GCNPat< 1499 (SIbuffer_atomic_cmpswap 1500 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1501 i32:$voffset, i32:$soffset, timm:$offset, 1502 timm:$cachepolicy, timm), 1503 (EXTRACT_SUBREG 1504 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1505 (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1), 1506 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1507 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_slc $cachepolicy)), 1508 sub0) 1509>; 1510 1511class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1512 PatFrag constant_ld> : GCNPat < 1513 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1514 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1515 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1516 >; 1517 1518multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1519 ValueType vt, PatFrag atomic_ld> { 1520 def : GCNPat < 1521 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1522 i16:$offset, i1:$slc))), 1523 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1524 >; 1525 1526 def : GCNPat < 1527 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1528 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1529 >; 1530} 1531 1532let SubtargetPredicate = isGFX6GFX7 in { 1533def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1534def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1535def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1536def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1537def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1538def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1539 1540defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1541defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1542} // End SubtargetPredicate = isGFX6GFX7 1543 1544multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1545 PatFrag ld> { 1546 1547 def : GCNPat < 1548 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1549 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))), 1550 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1551 >; 1552} 1553 1554let OtherPredicates = [Has16BitInsts] in { 1555 1556defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1557defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1558defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1559defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1560defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1561defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1562 1563defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1564 1565} // End OtherPredicates = [Has16BitInsts] 1566 1567multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1568 MUBUF_Pseudo InstrOffset, 1569 ValueType vt, PatFrag ld> { 1570 def : GCNPat < 1571 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1572 i32:$soffset, u16imm:$offset))), 1573 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1574 >; 1575 1576 def : GCNPat < 1577 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1578 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1579 >; 1580} 1581 1582// XXX - Is it possible to have a complex pattern in a PatFrag? 1583multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1584 MUBUF_Pseudo InstrOffset, 1585 ValueType vt, PatFrag ld_frag> { 1586 def : GCNPat < 1587 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1588 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1589 >; 1590 1591 def : GCNPat < 1592 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1593 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in) 1594 >; 1595} 1596 1597let OtherPredicates = [DisableFlatScratch] in { 1598defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1599defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1600defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1601defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1602defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1603defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1604defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1605defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1606defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1607defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1608 1609foreach vt = Reg32Types.types in { 1610defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>; 1611} 1612defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1613defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1614defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1615 1616let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1617defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1618defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1619defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1620defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1621defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1622defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1623 1624defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1625defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1626defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1627defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1628defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1629defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1630} 1631 1632} // End OtherPredicates = [DisableFlatScratch] 1633 1634multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1635 ValueType vt, PatFrag atomic_st> { 1636 // Store follows atomic op convention so address is first 1637 def : GCNPat < 1638 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1639 i16:$offset, i1:$slc), vt:$val), 1640 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0) 1641 >; 1642 1643 def : GCNPat < 1644 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1645 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0) 1646 >; 1647} 1648let SubtargetPredicate = isGFX6GFX7 in { 1649defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; 1650defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; 1651} // End Predicates = isGFX6GFX7 1652 1653 1654multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1655 PatFrag st> { 1656 1657 def : GCNPat < 1658 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1659 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)), 1660 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz) 1661 >; 1662} 1663 1664defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1665defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1666 1667multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1668 MUBUF_Pseudo InstrOffset, 1669 ValueType vt, PatFrag st, 1670 RegisterClass rc = VGPR_32> { 1671 def : GCNPat < 1672 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1673 i32:$soffset, u16imm:$offset)), 1674 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1675 >; 1676 1677 def : GCNPat < 1678 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1679 u16imm:$offset)), 1680 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0) 1681 >; 1682} 1683 1684let OtherPredicates = [DisableFlatScratch] in { 1685defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1686defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1687defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1688defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1689 1690foreach vt = Reg32Types.types in { 1691defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>; 1692} 1693 1694defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1695defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1696defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1697 1698 1699let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in { 1700 // Hiding the extract high pattern in the PatFrag seems to not 1701 // automatically increase the complexity. 1702let AddedComplexity = 1 in { 1703defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1704defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1705} 1706} 1707} // End OtherPredicates = [DisableFlatScratch] 1708 1709//===----------------------------------------------------------------------===// 1710// MTBUF Patterns 1711//===----------------------------------------------------------------------===// 1712 1713//===----------------------------------------------------------------------===// 1714// tbuffer_load/store_format patterns 1715//===----------------------------------------------------------------------===// 1716 1717multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1718 string opcode, ValueType memoryVt = vt> { 1719 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mtbuf_intrinsic_load<name, memoryVt>); 1720 1721 def : GCNPat< 1722 (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1723 timm:$format, timm:$auxiliary, 0)), 1724 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1725 (as_i8timm $format), 1726 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1727 (extract_swz $auxiliary)) 1728 >; 1729 1730 def : GCNPat< 1731 (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1732 timm:$format, timm:$auxiliary, timm)), 1733 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1734 (as_i8timm $format), 1735 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1736 (extract_swz $auxiliary)) 1737 >; 1738 1739 def : GCNPat< 1740 (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1741 timm:$format, timm:$auxiliary, 0)), 1742 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1743 (as_i8timm $format), 1744 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1745 (extract_swz $auxiliary)) 1746 >; 1747 1748 def : GCNPat< 1749 (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset, 1750 timm:$format, timm:$auxiliary, timm)), 1751 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1752 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1753 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), 1754 (as_i8timm $format), 1755 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1756 (extract_swz $auxiliary)) 1757 >; 1758} 1759 1760defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1761defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1762defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1763defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1764defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1765defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1766defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1767defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1768 1769let SubtargetPredicate = HasUnpackedD16VMem in { 1770 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1771 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1772 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1773 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">; 1774 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1775} // End HasUnpackedD16VMem. 1776 1777let SubtargetPredicate = HasPackedD16VMem in { 1778 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1779 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32, "TBUFFER_LOAD_FORMAT_D16_X">; 1780 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1781 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>; 1782 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1783} // End HasPackedD16VMem. 1784 1785multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1786 string opcode, ValueType memoryVt = vt> { 1787 defvar st = !if(!eq(!cast<string>(memoryVt), !cast<string>(vt)), name, mtbuf_intrinsic_store<name, memoryVt>); 1788 1789 def : GCNPat< 1790 (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset, 1791 timm:$format, timm:$auxiliary, 0), 1792 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, 1793 (as_i16timm $offset), (as_i8timm $format), 1794 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1795 (extract_swz $auxiliary)) 1796 >; 1797 1798 def : GCNPat< 1799 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset, 1800 timm:$format, timm:$auxiliary, timm), 1801 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, 1802 (as_i16timm $offset), (as_i8timm $format), 1803 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1804 (extract_swz $auxiliary)) 1805 >; 1806 1807 def : GCNPat< 1808 (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset, 1809 timm:$format, timm:$auxiliary, 0), 1810 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, 1811 (as_i16timm $offset), (as_i8timm $format), 1812 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1813 (extract_swz $auxiliary)) 1814 >; 1815 1816 def : GCNPat< 1817 (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1818 timm:$offset, timm:$format, timm:$auxiliary, timm), 1819 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1820 getVregSrcForVT<vt>.ret:$vdata, 1821 (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1), 1822 SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format), 1823 (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary), 1824 (extract_swz $auxiliary)) 1825 >; 1826} 1827 1828defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1829defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1830defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1831defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1832defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1833defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1834defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1835defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1836 1837let SubtargetPredicate = HasUnpackedD16VMem in { 1838 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1839 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1840 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1841 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">; 1842 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1843} // End HasUnpackedD16VMem. 1844 1845let SubtargetPredicate = HasPackedD16VMem in { 1846 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1847 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32, "TBUFFER_STORE_FORMAT_D16_X">; 1848 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1849 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>; 1850 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1851} // End HasPackedD16VMem. 1852 1853//===----------------------------------------------------------------------===// 1854// Target-specific instruction encodings. 1855//===----------------------------------------------------------------------===// 1856 1857//===----------------------------------------------------------------------===// 1858// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1859//===----------------------------------------------------------------------===// 1860 1861class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1862 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1863 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1864 let Inst{12} = ps.offen; 1865 let Inst{13} = ps.idxen; 1866 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1867 let Inst{16} = ps.lds; 1868 let Inst{24-18} = op; 1869 let Inst{31-26} = 0x38; 1870 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1871 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1872 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1873 let Inst{54} = !if(ps.has_slc, slc, ?); 1874 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1875 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1876} 1877 1878class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1879 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1880 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1881 let Inst{25} = op{7}; 1882} 1883 1884class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1885 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1886 let Inst{15} = ps.addr64; 1887} 1888 1889//===----------------------------------------------------------------------===// 1890// MUBUF - GFX10. 1891//===----------------------------------------------------------------------===// 1892 1893let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1894 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1895 string asmName> { 1896 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1897 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1898 let AsmString = asmName # ps.AsmOperands; 1899 } 1900 } 1901 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1902 def _BOTHEN_gfx10 : 1903 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1904 def _IDXEN_gfx10 : 1905 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1906 def _OFFEN_gfx10 : 1907 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1908 def _OFFSET_gfx10 : 1909 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1910 } 1911 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1912 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1913 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1914 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1915 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1916 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1917 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1918 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1919 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1920 1921 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1922 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1923 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1924 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1925 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1926 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1927 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1928 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1929 } 1930 multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> { 1931 def _BOTHEN_RTN_gfx10 : 1932 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1933 def _IDXEN_RTN_gfx10 : 1934 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1935 def _OFFEN_RTN_gfx10 : 1936 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1937 def _OFFSET_RTN_gfx10 : 1938 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1939 } 1940 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1941 MUBUF_Real_AllAddr_gfx10<op>, MUBUF_Real_Atomics_RTN_gfx10<op>; 1942} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1943 1944defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1945defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1946defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1947defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1948defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1949defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1950defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1951defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1952// FIXME-GFX10: Add following instructions: 1953//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1954//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1955defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1956defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1957defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1958defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1959defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1960defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1961defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1962defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1963 1964def BUFFER_GL0_INV_gfx10 : 1965 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1966def BUFFER_GL1_INV_gfx10 : 1967 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 1968 1969//===----------------------------------------------------------------------===// 1970// MUBUF - GFX6, GFX7, GFX10. 1971//===----------------------------------------------------------------------===// 1972 1973let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 1974 multiclass MUBUF_Real_gfx6<bits<8> op> { 1975 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1976 } 1977} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 1978 1979let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 1980 multiclass MUBUF_Real_gfx7<bits<8> op> { 1981 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1982 } 1983} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 1984 1985let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 1986 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 1987 def _ADDR64_gfx6_gfx7 : 1988 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1989 def _BOTHEN_gfx6_gfx7 : 1990 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1991 def _IDXEN_gfx6_gfx7 : 1992 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1993 def _OFFEN_gfx6_gfx7 : 1994 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1995 def _OFFSET_gfx6_gfx7 : 1996 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1997 } 1998 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 1999 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2000 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 2001 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 2002 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 2003 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2004 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 2005 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2006 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 2007 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2008 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 2009 2010 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2011 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 2012 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 2013 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 2014 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2015 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 2016 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2017 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 2018 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2019 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 2020 } 2021 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 2022 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 2023 def _ADDR64_RTN_gfx6_gfx7 : 2024 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 2025 def _BOTHEN_RTN_gfx6_gfx7 : 2026 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2027 def _IDXEN_RTN_gfx6_gfx7 : 2028 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2029 def _OFFEN_RTN_gfx6_gfx7 : 2030 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2031 def _OFFSET_RTN_gfx6_gfx7 : 2032 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2033 } 2034} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2035 2036multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 2037 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 2038 2039multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 2040 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 2041 2042multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 2043 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 2044 2045// FIXME-GFX6: Following instructions are available only on GFX6. 2046//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 2047//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 2048 2049defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 2050defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2051defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2052defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2053defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2054defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2055defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2056defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2057defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 2058defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 2059defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 2060defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 2061defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 2062defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 2063defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 2064defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 2065defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 2066defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 2067defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 2068defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 2069defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 2070defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 2071 2072defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 2073defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 2074defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 2075defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 2076defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 2077defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 2078defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 2079defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 2080defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 2081defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 2082defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 2083defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 2084defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 2085defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 2086defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 2087defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 2088defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 2089defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 2090defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 2091defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 2092defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 2093defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 2094defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 2095defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 2096defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 2097defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 2098defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 2099defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 2100defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 2101// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 2102defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 2103defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 2104defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 2105 2106defm BUFFER_ATOMIC_CSUB : MUBUF_Real_Atomics_RTN_gfx10<0x034>; 2107 2108defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 2109defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 2110def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 2111 2112//===----------------------------------------------------------------------===// 2113// Base ENC_MTBUF for GFX6, GFX7, GFX10. 2114//===----------------------------------------------------------------------===// 2115 2116class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 2117 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 2118 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2119 let Inst{12} = ps.offen; 2120 let Inst{13} = ps.idxen; 2121 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2122 let Inst{18-16} = op; 2123 let Inst{31-26} = 0x3a; //encoding 2124 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2125 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2126 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2127 let Inst{54} = !if(ps.has_slc, slc, ?); 2128 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2129 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2130} 2131 2132//===----------------------------------------------------------------------===// 2133// MTBUF - GFX10. 2134//===----------------------------------------------------------------------===// 2135 2136class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2137 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2138 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2139 let Inst{25-19} = format; 2140 let Inst{53} = op{3}; 2141} 2142 2143let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2144 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2145 def _BOTHEN_gfx10 : 2146 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2147 def _IDXEN_gfx10 : 2148 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2149 def _OFFEN_gfx10 : 2150 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2151 def _OFFSET_gfx10 : 2152 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2153 } 2154} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2155 2156defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2157defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2158defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2159defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2160defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2161defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2162defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2163defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2164 2165//===----------------------------------------------------------------------===// 2166// MTBUF - GFX6, GFX7, GFX10. 2167//===----------------------------------------------------------------------===// 2168 2169class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2170 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2171 let Inst{15} = ps.addr64; 2172 let Inst{22-19} = dfmt; 2173 let Inst{25-23} = nfmt; 2174} 2175 2176let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2177 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2178 def _ADDR64_gfx6_gfx7 : 2179 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2180 def _BOTHEN_gfx6_gfx7 : 2181 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2182 def _IDXEN_gfx6_gfx7 : 2183 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2184 def _OFFEN_gfx6_gfx7 : 2185 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2186 def _OFFSET_gfx6_gfx7 : 2187 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2188 } 2189} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2190 2191multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2192 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2193 2194defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2195defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2196defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2197defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2198defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2199defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2200defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2201defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2202 2203//===----------------------------------------------------------------------===// 2204// GFX8, GFX9 (VI). 2205//===----------------------------------------------------------------------===// 2206 2207class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2208 MUBUF_Real<ps>, 2209 Enc64, 2210 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2211 let AssemblerPredicate = isGFX8GFX9; 2212 let DecoderNamespace = "GFX8"; 2213 2214 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2215 let Inst{12} = ps.offen; 2216 let Inst{13} = ps.idxen; 2217 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2218 let Inst{16} = ps.lds; 2219 let Inst{17} = !if(ps.has_slc, slc, ?); 2220 let Inst{24-18} = op; 2221 let Inst{31-26} = 0x38; //encoding 2222 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2223 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2224 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2225 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2226 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2227} 2228 2229multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2230 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2231 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2232 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2233 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2234} 2235 2236multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2237 2238 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2239 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2240 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2241 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2242 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2243 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2244 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2245 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2246 2247 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2248 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2249 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2250 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2251 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2252 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2253 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2254 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2255} 2256 2257class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2258 MUBUF_Real<ps>, 2259 Enc64, 2260 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2261 let AssemblerPredicate=HasUnpackedD16VMem; 2262 let DecoderNamespace="GFX80_UNPACKED"; 2263 2264 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2265 let Inst{12} = ps.offen; 2266 let Inst{13} = ps.idxen; 2267 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2268 let Inst{16} = ps.lds; 2269 let Inst{17} = !if(ps.has_slc, slc, ?); 2270 let Inst{24-18} = op; 2271 let Inst{31-26} = 0x38; //encoding 2272 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2273 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2274 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2275 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2276 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2277} 2278 2279multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2280 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2281 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2282 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2283 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2284} 2285 2286multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2287 MUBUF_Real_AllAddr_vi<op> { 2288 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2289 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2290 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2291 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2292} 2293 2294defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2295defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2296defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2297defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2298defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2299defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2300defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2301defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2302let SubtargetPredicate = HasUnpackedD16VMem in { 2303 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2304 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2305 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2306 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2307 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2308 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2309 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2310 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2311} // End HasUnpackedD16VMem. 2312let SubtargetPredicate = HasPackedD16VMem in { 2313 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2314 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2315 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2316 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2317 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2318 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2319 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2320 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2321} // End HasPackedD16VMem. 2322defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2323defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2324defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2325defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2326defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2327defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2328defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2329defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2330defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2331defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2332defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2333defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2334defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2335defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2336defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2337defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2338 2339defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2340defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2341defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2342defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2343defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2344defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2345 2346defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2347defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2348 2349defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2350defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2351defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2352defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2353defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2354defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2355defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2356defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2357defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2358defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2359defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2360defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2361defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2362 2363defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2364defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2365defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2366defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2367defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2368defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2369defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2370defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2371defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2372defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2373defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2374defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2375defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2376 2377def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2378 2379def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2380def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2381 2382let SubtargetPredicate = HasAtomicFaddInsts in { 2383 2384defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_AllAddr_vi <0x4d>; 2385defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>; 2386 2387} // End SubtargetPredicate = HasAtomicFaddInsts 2388 2389class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2390 MTBUF_Real<ps>, 2391 Enc64, 2392 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2393 let AssemblerPredicate = isGFX8GFX9; 2394 let DecoderNamespace = "GFX8"; 2395 2396 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2397 let Inst{12} = ps.offen; 2398 let Inst{13} = ps.idxen; 2399 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2400 let Inst{18-15} = op; 2401 let Inst{22-19} = dfmt; 2402 let Inst{25-23} = nfmt; 2403 let Inst{31-26} = 0x3a; //encoding 2404 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2405 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2406 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2407 let Inst{54} = !if(ps.has_slc, slc, ?); 2408 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2409 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2410} 2411 2412multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2413 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2414 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2415 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2416 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2417} 2418 2419class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2420 MTBUF_Real<ps>, 2421 Enc64, 2422 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2423 let AssemblerPredicate=HasUnpackedD16VMem; 2424 let DecoderNamespace="GFX80_UNPACKED"; 2425 2426 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2427 let Inst{12} = ps.offen; 2428 let Inst{13} = ps.idxen; 2429 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2430 let Inst{18-15} = op; 2431 let Inst{22-19} = dfmt; 2432 let Inst{25-23} = nfmt; 2433 let Inst{31-26} = 0x3a; //encoding 2434 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2435 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2436 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2437 let Inst{54} = !if(ps.has_slc, slc, ?); 2438 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2439 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2440} 2441 2442multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2443 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2444 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2445 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2446 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2447} 2448 2449defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2450defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2451defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2452defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2453defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2454defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2455defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2456defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2457let SubtargetPredicate = HasUnpackedD16VMem in { 2458 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2459 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2460 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2461 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2462 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2463 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2464 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2465 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2466} // End HasUnpackedD16VMem. 2467let SubtargetPredicate = HasPackedD16VMem in { 2468 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2469 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2470 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2471 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2472 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2473 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2474 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2475 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2476} // End HasUnpackedD16VMem. 2477 2478def MUBUFInfoTable : GenericTable { 2479 let FilterClass = "MUBUF_Pseudo"; 2480 let CppTypeName = "MUBUFInfo"; 2481 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2482 2483 let PrimaryKey = ["Opcode"]; 2484 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2485} 2486 2487def getMUBUFInfoFromOpcode : SearchIndex { 2488 let Table = MUBUFInfoTable; 2489 let Key = ["Opcode"]; 2490} 2491 2492def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2493 let Table = MUBUFInfoTable; 2494 let Key = ["BaseOpcode", "elements"]; 2495} 2496 2497def MTBUFInfoTable : GenericTable { 2498 let FilterClass = "MTBUF_Pseudo"; 2499 let CppTypeName = "MTBUFInfo"; 2500 let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"]; 2501 2502 let PrimaryKey = ["Opcode"]; 2503 let PrimaryKeyName = "getMTBUFOpcodeHelper"; 2504} 2505 2506def getMTBUFInfoFromOpcode : SearchIndex { 2507 let Table = MTBUFInfoTable; 2508 let Key = ["Opcode"]; 2509} 2510 2511def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex { 2512 let Table = MTBUFInfoTable; 2513 let Key = ["BaseOpcode", "elements"]; 2514} 2515