1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 10def MUBUFAddr64 : ComplexPattern<i64, 8, "SelectMUBUFAddr64">; 11def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 12 13def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 14def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 15 16def MUBUFOffset : ComplexPattern<i64, 7, "SelectMUBUFOffset">; 17def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 18def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 19 20def BUFAddrKind { 21 int Offset = 0; 22 int OffEn = 1; 23 int IdxEn = 2; 24 int BothEn = 3; 25 int Addr64 = 4; 26} 27 28class getAddrName<int addrKind> { 29 string ret = 30 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 31 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 32 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 33 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 34 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 35 ""))))); 36} 37 38class MUBUFAddr64Table <bit is_addr64, string Name> { 39 bit IsAddr64 = is_addr64; 40 string OpName = Name; 41} 42 43class MUBUFLdsTable <bit is_lds, string Name> { 44 bit IsLds = is_lds; 45 string OpName = Name; 46} 47 48class MTBUFAddr64Table <bit is_addr64, string Name> { 49 bit IsAddr64 = is_addr64; 50 string OpName = Name; 51} 52 53//===----------------------------------------------------------------------===// 54// MTBUF classes 55//===----------------------------------------------------------------------===// 56 57class MTBUF_Pseudo <string opName, dag outs, dag ins, 58 string asmOps, list<dag> pattern=[]> : 59 InstSI<outs, ins, "", pattern>, 60 SIMCInstr<opName, SIEncodingFamily.NONE> { 61 62 let isPseudo = 1; 63 let isCodeGenOnly = 1; 64 let Size = 8; 65 let UseNamedOperandTable = 1; 66 67 string Mnemonic = opName; 68 string AsmOperands = asmOps; 69 70 let VM_CNT = 1; 71 let EXP_CNT = 1; 72 let MTBUF = 1; 73 let Uses = [EXEC]; 74 let hasSideEffects = 0; 75 let SchedRW = [WriteVMEM]; 76 77 let AsmMatchConverter = "cvtMtbuf"; 78 79 bits<1> offen = 0; 80 bits<1> idxen = 0; 81 bits<1> addr64 = 0; 82 bits<1> has_vdata = 1; 83 bits<1> has_vaddr = 1; 84 bits<1> has_glc = 1; 85 bits<1> has_dlc = 1; 86 bits<1> glc_value = 0; // the value for glc if no such operand 87 bits<1> dlc_value = 0; // the value for dlc if no such operand 88 bits<1> has_srsrc = 1; 89 bits<1> has_soffset = 1; 90 bits<1> has_offset = 1; 91 bits<1> has_slc = 1; 92 bits<1> has_tfe = 1; 93} 94 95class MTBUF_Real <MTBUF_Pseudo ps> : 96 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 97 98 let isPseudo = 0; 99 let isCodeGenOnly = 0; 100 101 // copy relevant pseudo op flags 102 let SubtargetPredicate = ps.SubtargetPredicate; 103 let AsmMatchConverter = ps.AsmMatchConverter; 104 let Constraints = ps.Constraints; 105 let DisableEncoding = ps.DisableEncoding; 106 let TSFlags = ps.TSFlags; 107 108 bits<12> offset; 109 bits<1> glc; 110 bits<1> dlc; 111 bits<7> format; 112 bits<8> vaddr; 113 bits<8> vdata; 114 bits<7> srsrc; 115 bits<1> slc; 116 bits<1> tfe; 117 bits<8> soffset; 118 119 bits<4> dfmt = format{3-0}; 120 bits<3> nfmt = format{6-4}; 121} 122 123class getMTBUFInsDA<list<RegisterClass> vdataList, 124 list<RegisterClass> vaddrList=[]> { 125 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 126 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 127 dag InsNoData = !if(!empty(vaddrList), 128 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 129 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc), 130 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 131 offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc) 132 ); 133 dag InsData = !if(!empty(vaddrList), 134 (ins vdataClass:$vdata, SReg_128:$srsrc, 135 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 136 SLC:$slc, TFE:$tfe, DLC:$dlc), 137 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 138 SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc, 139 SLC:$slc, TFE:$tfe, DLC:$dlc) 140 ); 141 dag ret = !if(!empty(vdataList), InsNoData, InsData); 142} 143 144class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 145 dag ret = 146 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 147 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 148 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 149 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 150 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 151 (ins)))))); 152} 153 154class getMTBUFAsmOps<int addrKind> { 155 string Pfx = 156 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset", 157 !if(!eq(addrKind, BUFAddrKind.OffEn), 158 "$vaddr, $srsrc, $format, $soffset offen", 159 !if(!eq(addrKind, BUFAddrKind.IdxEn), 160 "$vaddr, $srsrc, $format, $soffset idxen", 161 !if(!eq(addrKind, BUFAddrKind.BothEn), 162 "$vaddr, $srsrc, $format, $soffset idxen offen", 163 !if(!eq(addrKind, BUFAddrKind.Addr64), 164 "$vaddr, $srsrc, $format, $soffset addr64", 165 ""))))); 166 string ret = Pfx # "$offset"; 167} 168 169class MTBUF_SetupAddr<int addrKind> { 170 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 171 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 172 173 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 174 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 175 176 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 177 178 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 179} 180 181class MTBUF_Load_Pseudo <string opName, 182 int addrKind, 183 RegisterClass vdataClass, 184 list<dag> pattern=[], 185 // Workaround bug bz30254 186 int addrKindCopy = addrKind> 187 : MTBUF_Pseudo<opName, 188 (outs vdataClass:$vdata), 189 getMTBUFIns<addrKindCopy>.ret, 190 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc", 191 pattern>, 192 MTBUF_SetupAddr<addrKindCopy> { 193 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 194 let mayLoad = 1; 195 let mayStore = 0; 196} 197 198multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 199 ValueType load_vt = i32, 200 SDPatternOperator ld = null_frag> { 201 202 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 203 [(set load_vt:$vdata, 204 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format, 205 i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>, 206 MTBUFAddr64Table<0, NAME>; 207 208 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 209 [(set load_vt:$vdata, 210 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 211 i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>, 212 MTBUFAddr64Table<1, NAME>; 213 214 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 215 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 216 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 217 218 let DisableWQM = 1 in { 219 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 220 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 221 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 222 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 223 } 224} 225 226class MTBUF_Store_Pseudo <string opName, 227 int addrKind, 228 RegisterClass vdataClass, 229 list<dag> pattern=[], 230 // Workaround bug bz30254 231 int addrKindCopy = addrKind, 232 RegisterClass vdataClassCopy = vdataClass> 233 : MTBUF_Pseudo<opName, 234 (outs), 235 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 236 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc", 237 pattern>, 238 MTBUF_SetupAddr<addrKindCopy> { 239 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 240 let mayLoad = 0; 241 let mayStore = 1; 242} 243 244multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 245 ValueType store_vt = i32, 246 SDPatternOperator st = null_frag> { 247 248 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 249 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 250 i16:$offset, i8:$format, i1:$glc, 251 i1:$slc, i1:$tfe, i1:$dlc))]>, 252 MTBUFAddr64Table<0, NAME>; 253 254 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 255 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 256 i16:$offset, i8:$format, i1:$glc, 257 i1:$slc, i1:$tfe, i1:$dlc))]>, 258 MTBUFAddr64Table<1, NAME>; 259 260 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 261 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 262 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 263 264 let DisableWQM = 1 in { 265 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 266 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 267 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 268 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 269 } 270} 271 272 273//===----------------------------------------------------------------------===// 274// MUBUF classes 275//===----------------------------------------------------------------------===// 276 277class MUBUFGetBaseOpcode<string Op> { 278 string ret = !subst("DWORDX2", "DWORD", 279 !subst("DWORDX3", "DWORD", 280 !subst("DWORDX4", "DWORD", Op))); 281} 282 283class MUBUF_Pseudo <string opName, dag outs, dag ins, 284 string asmOps, list<dag> pattern=[]> : 285 InstSI<outs, ins, "", pattern>, 286 SIMCInstr<opName, SIEncodingFamily.NONE> { 287 288 let isPseudo = 1; 289 let isCodeGenOnly = 1; 290 let Size = 8; 291 let UseNamedOperandTable = 1; 292 293 string Mnemonic = opName; 294 string AsmOperands = asmOps; 295 296 Instruction Opcode = !cast<Instruction>(NAME); 297 Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret); 298 299 let VM_CNT = 1; 300 let EXP_CNT = 1; 301 let MUBUF = 1; 302 let Uses = [EXEC]; 303 let hasSideEffects = 0; 304 let SchedRW = [WriteVMEM]; 305 306 let AsmMatchConverter = "cvtMubuf"; 307 308 bits<1> offen = 0; 309 bits<1> idxen = 0; 310 bits<1> addr64 = 0; 311 bits<1> lds = 0; 312 bits<1> has_vdata = 1; 313 bits<1> has_vaddr = 1; 314 bits<1> has_glc = 1; 315 bits<1> has_dlc = 1; 316 bits<1> glc_value = 0; // the value for glc if no such operand 317 bits<1> dlc_value = 0; // the value for dlc if no such operand 318 bits<1> has_srsrc = 1; 319 bits<1> has_soffset = 1; 320 bits<1> has_offset = 1; 321 bits<1> has_slc = 1; 322 bits<1> has_tfe = 1; 323 bits<4> dwords = 0; 324} 325 326class MUBUF_Real <MUBUF_Pseudo ps> : 327 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 328 329 let isPseudo = 0; 330 let isCodeGenOnly = 0; 331 332 // copy relevant pseudo op flags 333 let SubtargetPredicate = ps.SubtargetPredicate; 334 let AsmMatchConverter = ps.AsmMatchConverter; 335 let Constraints = ps.Constraints; 336 let DisableEncoding = ps.DisableEncoding; 337 let TSFlags = ps.TSFlags; 338 339 bits<12> offset; 340 bits<1> glc; 341 bits<1> dlc; 342 bits<8> vaddr; 343 bits<8> vdata; 344 bits<7> srsrc; 345 bits<1> slc; 346 bits<1> tfe; 347 bits<8> soffset; 348} 349 350 351// For cache invalidation instructions. 352class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> : 353 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 354 355 let AsmMatchConverter = ""; 356 357 let hasSideEffects = 1; 358 let mayStore = 1; 359 360 // Set everything to 0. 361 let offen = 0; 362 let idxen = 0; 363 let addr64 = 0; 364 let has_vdata = 0; 365 let has_vaddr = 0; 366 let has_glc = 0; 367 let has_dlc = 0; 368 let glc_value = 0; 369 let dlc_value = 0; 370 let has_srsrc = 0; 371 let has_soffset = 0; 372 let has_offset = 0; 373 let has_slc = 0; 374 let has_tfe = 0; 375} 376 377class getMUBUFInsDA<list<RegisterClass> vdataList, 378 list<RegisterClass> vaddrList=[], 379 bit isLds = 0> { 380 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 381 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 382 dag InsNoData = !if(!empty(vaddrList), 383 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 384 offset:$offset, GLC:$glc, SLC:$slc), 385 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 386 offset:$offset, GLC:$glc, SLC:$slc) 387 ); 388 dag InsData = !if(!empty(vaddrList), 389 (ins vdataClass:$vdata, SReg_128:$srsrc, 390 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 391 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 392 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 393 ); 394 dag ret = !con( 395 !if(!empty(vdataList), InsNoData, InsData), 396 !if(isLds, (ins DLC:$dlc), (ins TFE:$tfe, DLC:$dlc)) 397 ); 398} 399 400class getMUBUFDwords<RegisterClass regClass> { 401 string regClassAsInt = !cast<string>(regClass); 402 int ret = 403 !if(!eq(regClassAsInt, !cast<string>(VGPR_32)), 1, 404 !if(!eq(regClassAsInt, !cast<string>(VReg_64)), 2, 405 !if(!eq(regClassAsInt, !cast<string>(VReg_96)), 3, 406 !if(!eq(regClassAsInt, !cast<string>(VReg_128)), 4, 407 0)))); 408} 409 410class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 411 dag ret = 412 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 413 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 414 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 415 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 416 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 417 (ins)))))); 418} 419 420class getMUBUFAsmOps<int addrKind> { 421 string Pfx = 422 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 423 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 424 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 425 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 426 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 427 ""))))); 428 string ret = Pfx # "$offset"; 429} 430 431class MUBUF_SetupAddr<int addrKind> { 432 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 433 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 434 435 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 436 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 437 438 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 439 440 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 441} 442 443class MUBUF_Load_Pseudo <string opName, 444 int addrKind, 445 RegisterClass vdataClass, 446 bit HasTiedDest = 0, 447 bit isLds = 0, 448 list<dag> pattern=[], 449 // Workaround bug bz30254 450 int addrKindCopy = addrKind> 451 : MUBUF_Pseudo<opName, 452 (outs vdataClass:$vdata), 453 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 454 !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 455 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 456 !if(isLds, " lds", "$tfe") # "$dlc", 457 pattern>, 458 MUBUF_SetupAddr<addrKindCopy> { 459 let PseudoInstr = opName # !if(isLds, "_lds", "") # 460 "_" # getAddrName<addrKindCopy>.ret; 461 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 462 463 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 464 let mayLoad = 1; 465 let mayStore = 0; 466 let maybeAtomic = 1; 467 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 468 let has_tfe = !if(isLds, 0, 1); 469 let lds = isLds; 470 let dwords = getMUBUFDwords<vdataClass>.ret; 471} 472 473class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat < 474 (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))), 475 (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)) 476>; 477 478class MUBUF_Addr64_Load_Pat <Instruction inst, 479 ValueType load_vt = i32, 480 SDPatternOperator ld = null_frag> : Pat < 481 (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))), 482 (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)) 483>; 484 485multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> { 486 def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>; 487 def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>; 488} 489 490 491// FIXME: tfe can't be an operand because it requires a separate 492// opcode because it needs an N+1 register class dest register. 493multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 494 ValueType load_vt = i32, 495 SDPatternOperator ld = null_frag, 496 bit TiedDest = 0, 497 bit isLds = 0> { 498 499 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>, 500 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 501 502 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, TiedDest, isLds>, 503 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 504 505 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 506 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 507 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 508 509 let DisableWQM = 1 in { 510 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>; 511 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 512 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 513 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 514 } 515} 516 517multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass, 518 ValueType load_vt = i32, 519 SDPatternOperator ld_nolds = null_frag, 520 SDPatternOperator ld_lds = null_frag> { 521 defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>; 522 defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>; 523} 524 525class MUBUF_Store_Pseudo <string opName, 526 int addrKind, 527 RegisterClass vdataClass, 528 list<dag> pattern=[], 529 // Workaround bug bz30254 530 int addrKindCopy = addrKind, 531 RegisterClass vdataClassCopy = vdataClass> 532 : MUBUF_Pseudo<opName, 533 (outs), 534 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 535 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc", 536 pattern>, 537 MUBUF_SetupAddr<addrKindCopy> { 538 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 539 let mayLoad = 0; 540 let mayStore = 1; 541 let maybeAtomic = 1; 542 let dwords = getMUBUFDwords<vdataClass>.ret; 543} 544 545multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 546 ValueType store_vt = i32, 547 SDPatternOperator st = null_frag> { 548 549 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 550 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 551 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>, 552 MUBUFAddr64Table<0, NAME>; 553 554 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 555 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 556 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>, 557 MUBUFAddr64Table<1, NAME>; 558 559 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 560 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 561 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 562 563 let DisableWQM = 1 in { 564 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 565 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 566 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 567 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 568 } 569} 570 571class MUBUF_Pseudo_Store_Lds<string opName> 572 : MUBUF_Pseudo<opName, 573 (outs), 574 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 575 " $srsrc, $soffset$offset lds$glc$slc"> { 576 let mayLoad = 0; 577 let mayStore = 1; 578 let maybeAtomic = 1; 579 580 let has_vdata = 0; 581 let has_vaddr = 0; 582 let has_tfe = 0; 583 let lds = 1; 584 585 let Uses = [EXEC, M0]; 586 let AsmMatchConverter = "cvtMubufLds"; 587} 588 589class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 590 list<RegisterClass> vaddrList=[]> { 591 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 592 dag ret = !if(vdata_in, 593 !if(!empty(vaddrList), 594 (ins vdataClass:$vdata_in, 595 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 596 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 597 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 598 ), 599 !if(!empty(vaddrList), 600 (ins vdataClass:$vdata, 601 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 602 (ins vdataClass:$vdata, vaddrClass:$vaddr, 603 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 604 )); 605} 606 607class getMUBUFAtomicIns<int addrKind, 608 RegisterClass vdataClass, 609 bit vdata_in, 610 // Workaround bug bz30254 611 RegisterClass vdataClassCopy=vdataClass> { 612 dag ret = 613 !if(!eq(addrKind, BUFAddrKind.Offset), 614 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 615 !if(!eq(addrKind, BUFAddrKind.OffEn), 616 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 617 !if(!eq(addrKind, BUFAddrKind.IdxEn), 618 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 619 !if(!eq(addrKind, BUFAddrKind.BothEn), 620 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 621 !if(!eq(addrKind, BUFAddrKind.Addr64), 622 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 623 (ins)))))); 624} 625 626class MUBUF_Atomic_Pseudo<string opName, 627 int addrKind, 628 dag outs, 629 dag ins, 630 string asmOps, 631 list<dag> pattern=[], 632 // Workaround bug bz30254 633 int addrKindCopy = addrKind> 634 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 635 MUBUF_SetupAddr<addrKindCopy> { 636 let mayStore = 1; 637 let mayLoad = 1; 638 let hasPostISelHook = 1; 639 let hasSideEffects = 1; 640 let DisableWQM = 1; 641 let has_glc = 0; 642 let has_dlc = 0; 643 let has_tfe = 0; 644 let maybeAtomic = 1; 645} 646 647class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 648 RegisterClass vdataClass, 649 list<dag> pattern=[], 650 // Workaround bug bz30254 651 int addrKindCopy = addrKind, 652 RegisterClass vdataClassCopy = vdataClass> 653 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 654 (outs), 655 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 656 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 657 pattern>, 658 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 659 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 660 let glc_value = 0; 661 let dlc_value = 0; 662 let AsmMatchConverter = "cvtMubufAtomic"; 663} 664 665class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 666 RegisterClass vdataClass, 667 list<dag> pattern=[], 668 // Workaround bug bz30254 669 int addrKindCopy = addrKind, 670 RegisterClass vdataClassCopy = vdataClass> 671 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 672 (outs vdataClassCopy:$vdata), 673 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 674 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 675 pattern>, 676 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 677 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 678 let glc_value = 1; 679 let dlc_value = 0; 680 let Constraints = "$vdata = $vdata_in"; 681 let DisableEncoding = "$vdata_in"; 682 let AsmMatchConverter = "cvtMubufAtomicReturn"; 683} 684 685multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName, 686 RegisterClass vdataClass, 687 ValueType vdataType, 688 SDPatternOperator atomic, 689 bit isFP = getIsFP<vdataType>.ret> { 690 let FPAtomic = isFP in 691 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 692 MUBUFAddr64Table <0, NAME>; 693 694 let FPAtomic = isFP in 695 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 696 MUBUFAddr64Table <1, NAME>; 697 698 let FPAtomic = isFP in 699 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 700 701 let FPAtomic = isFP in 702 703 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 704 705 let FPAtomic = isFP in 706 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 707} 708 709multiclass MUBUF_Pseudo_Atomics_RTN <string opName, 710 RegisterClass vdataClass, 711 ValueType vdataType, 712 SDPatternOperator atomic, 713 bit isFP = getIsFP<vdataType>.ret> { 714 let FPAtomic = isFP in 715 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 716 [(set vdataType:$vdata, 717 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 718 vdataType:$vdata_in))]>, 719 MUBUFAddr64Table <0, NAME # "_RTN">; 720 721 let FPAtomic = isFP in 722 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 723 [(set vdataType:$vdata, 724 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 725 vdataType:$vdata_in))]>, 726 MUBUFAddr64Table <1, NAME # "_RTN">; 727 728 let FPAtomic = isFP in 729 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 730 731 let FPAtomic = isFP in 732 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 733 734 let FPAtomic = isFP in 735 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 736} 737 738multiclass MUBUF_Pseudo_Atomics <string opName, 739 RegisterClass vdataClass, 740 ValueType vdataType, 741 SDPatternOperator atomic> : 742 MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>, 743 MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>; 744 745 746//===----------------------------------------------------------------------===// 747// MUBUF Instructions 748//===----------------------------------------------------------------------===// 749 750defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 751 "buffer_load_format_x", VGPR_32 752>; 753defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 754 "buffer_load_format_xy", VReg_64 755>; 756defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 757 "buffer_load_format_xyz", VReg_96 758>; 759defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 760 "buffer_load_format_xyzw", VReg_128 761>; 762defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 763 "buffer_store_format_x", VGPR_32 764>; 765defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 766 "buffer_store_format_xy", VReg_64 767>; 768defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 769 "buffer_store_format_xyz", VReg_96 770>; 771defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 772 "buffer_store_format_xyzw", VReg_128 773>; 774 775let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 776 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 777 "buffer_load_format_d16_x", VGPR_32 778 >; 779 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 780 "buffer_load_format_d16_xy", VReg_64 781 >; 782 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 783 "buffer_load_format_d16_xyz", VReg_96 784 >; 785 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 786 "buffer_load_format_d16_xyzw", VReg_128 787 >; 788 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 789 "buffer_store_format_d16_x", VGPR_32 790 >; 791 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 792 "buffer_store_format_d16_xy", VReg_64 793 >; 794 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 795 "buffer_store_format_d16_xyz", VReg_96 796 >; 797 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 798 "buffer_store_format_d16_xyzw", VReg_128 799 >; 800} // End HasUnpackedD16VMem. 801 802let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 803 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 804 "buffer_load_format_d16_x", VGPR_32 805 >; 806 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 807 "buffer_load_format_d16_xy", VGPR_32 808 >; 809 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 810 "buffer_load_format_d16_xyz", VReg_64 811 >; 812 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 813 "buffer_load_format_d16_xyzw", VReg_64 814 >; 815 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 816 "buffer_store_format_d16_x", VGPR_32 817 >; 818 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 819 "buffer_store_format_d16_xy", VGPR_32 820 >; 821 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 822 "buffer_store_format_d16_xyz", VReg_64 823 >; 824 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 825 "buffer_store_format_d16_xyzw", VReg_64 826 >; 827} // End HasPackedD16VMem. 828 829defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 830 "buffer_load_ubyte", VGPR_32, i32 831>; 832defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 833 "buffer_load_sbyte", VGPR_32, i32 834>; 835defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 836 "buffer_load_ushort", VGPR_32, i32 837>; 838defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 839 "buffer_load_sshort", VGPR_32, i32 840>; 841defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 842 "buffer_load_dword", VGPR_32, i32 843>; 844defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 845 "buffer_load_dwordx2", VReg_64, v2i32 846>; 847defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 848 "buffer_load_dwordx3", VReg_96, v3i32 849>; 850defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 851 "buffer_load_dwordx4", VReg_128, v4i32 852>; 853 854defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>; 855defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>; 856defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>; 857defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>; 858defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>; 859defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>; 860defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>; 861defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>; 862defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>; 863defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>; 864 865// This is not described in AMD documentation, 866// but 'lds' versions of these opcodes are available 867// in at least GFX8+ chips. See Bug 37653. 868let SubtargetPredicate = isGFX8GFX9 in { 869defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 870 "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1 871>; 872defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 873 "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1 874>; 875defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 876 "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1 877>; 878} 879 880defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 881 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 882>; 883defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 884 "buffer_store_short", VGPR_32, i32, truncstorei16_global 885>; 886defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 887 "buffer_store_dword", VGPR_32, i32, store_global 888>; 889defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 890 "buffer_store_dwordx2", VReg_64, v2i32, store_global 891>; 892defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 893 "buffer_store_dwordx3", VReg_96, v3i32, store_global 894>; 895defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 896 "buffer_store_dwordx4", VReg_128, v4i32, store_global 897>; 898defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 899 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 900>; 901defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 902 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 903>; 904defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 905 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 906>; 907defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 908 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 909>; 910defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 911 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 912>; 913defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 914 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 915>; 916defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 917 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 918>; 919defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 920 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 921>; 922defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 923 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 924>; 925defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 926 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 927>; 928defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 929 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 930>; 931defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 932 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 933>; 934defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 935 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 936>; 937defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 938 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 939>; 940defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 941 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 942>; 943defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 944 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 945>; 946defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 947 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 948>; 949defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 950 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 951>; 952defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 953 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 954>; 955defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 956 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 957>; 958defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 959 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 960>; 961defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 962 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 963>; 964defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 965 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 966>; 967defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 968 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 969>; 970defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 971 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 972>; 973defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 974 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 975>; 976 977let SubtargetPredicate = isGFX8GFX9 in { 978def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 979} 980 981let SubtargetPredicate = isGFX6 in { // isn't on CI & VI 982/* 983defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 984defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 985defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 986defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 987defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 988defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 989defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 990defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 991*/ 992 993def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 994 int_amdgcn_buffer_wbinvl1_sc>; 995} 996 997let SubtargetPredicate = HasD16LoadStore in { 998 999defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 1000 "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1 1001>; 1002 1003defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 1004 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 1005>; 1006 1007defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 1008 "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1 1009>; 1010 1011defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 1012 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 1013>; 1014 1015defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 1016 "buffer_load_short_d16", VGPR_32, i32, null_frag, 1 1017>; 1018 1019defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 1020 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 1021>; 1022 1023defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 1024 "buffer_store_byte_d16_hi", VGPR_32, i32 1025>; 1026 1027defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 1028 "buffer_store_short_d16_hi", VGPR_32, i32 1029>; 1030 1031defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 1032 "buffer_load_format_d16_hi_x", VGPR_32 1033>; 1034defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 1035 "buffer_store_format_d16_hi_x", VGPR_32 1036>; 1037 1038} // End HasD16LoadStore 1039 1040def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 1041 int_amdgcn_buffer_wbinvl1>; 1042 1043let SubtargetPredicate = HasAtomicFaddInsts in { 1044 1045defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN < 1046 "buffer_atomic_add_f32", VGPR_32, f32, atomic_add_global 1047>; 1048defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN < 1049 "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_add_global 1050>; 1051 1052} // End SubtargetPredicate = HasAtomicFaddInsts 1053 1054//===----------------------------------------------------------------------===// 1055// MTBUF Instructions 1056//===----------------------------------------------------------------------===// 1057 1058defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 1059defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 1060defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_96>; 1061defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 1062defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 1063defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 1064defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_96>; 1065defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 1066 1067let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 1068 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1069 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64>; 1070 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96>; 1071 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128>; 1072 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1073 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64>; 1074 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96>; 1075 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>; 1076} // End HasUnpackedD16VMem. 1077 1078let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1079 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1080 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32>; 1081 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64>; 1082 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64>; 1083 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1084 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32>; 1085 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64>; 1086 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>; 1087} // End HasPackedD16VMem. 1088 1089let SubtargetPredicate = isGFX7Plus in { 1090 1091//===----------------------------------------------------------------------===// 1092// Instruction definitions for CI and newer. 1093//===----------------------------------------------------------------------===// 1094 1095def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1096 int_amdgcn_buffer_wbinvl1_vol>; 1097 1098} // End let SubtargetPredicate = isGFX7Plus 1099 1100let SubtargetPredicate = isGFX10Plus in { 1101 def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">; 1102 def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">; 1103} // End SubtargetPredicate = isGFX10Plus 1104 1105//===----------------------------------------------------------------------===// 1106// MUBUF Patterns 1107//===----------------------------------------------------------------------===// 1108 1109def extract_glc : SDNodeXForm<imm, [{ 1110 return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8); 1111}]>; 1112 1113def extract_slc : SDNodeXForm<imm, [{ 1114 return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8); 1115}]>; 1116 1117def extract_dlc : SDNodeXForm<imm, [{ 1118 return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8); 1119}]>; 1120 1121//===----------------------------------------------------------------------===// 1122// buffer_load/store_format patterns 1123//===----------------------------------------------------------------------===// 1124 1125multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1126 string opcode> { 1127 def : GCNPat< 1128 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1129 imm:$cachepolicy, 0)), 1130 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1131 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1132 >; 1133 1134 def : GCNPat< 1135 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1136 imm:$cachepolicy, 0)), 1137 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1138 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1139 >; 1140 1141 def : GCNPat< 1142 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1143 imm:$cachepolicy, imm)), 1144 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1145 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1146 >; 1147 1148 def : GCNPat< 1149 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1150 imm:$cachepolicy, imm)), 1151 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1152 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1153 $rsrc, $soffset, (as_i16imm $offset), 1154 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1155 >; 1156} 1157 1158defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1159defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">; 1160defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1161defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">; 1162defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">; 1163defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">; 1164defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1165defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">; 1166 1167let SubtargetPredicate = HasUnpackedD16VMem in { 1168 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1169 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1170 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1171 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1172} // End HasUnpackedD16VMem. 1173 1174let SubtargetPredicate = HasPackedD16VMem in { 1175 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1176 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">; 1177 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1178 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">; 1179 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1180 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1181} // End HasPackedD16VMem. 1182 1183defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1184defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">; 1185defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1186defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">; 1187defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">; 1188defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">; 1189defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1190defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">; 1191defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">; 1192defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">; 1193defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">; 1194defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort, i32, "BUFFER_LOAD_USHORT">; 1195 1196multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1197 string opcode> { 1198 def : GCNPat< 1199 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1200 imm:$cachepolicy, 0), 1201 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 1202 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1203 >; 1204 1205 def : GCNPat< 1206 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1207 imm:$cachepolicy, 0), 1208 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1209 (as_i16imm $offset), (extract_glc $cachepolicy), 1210 (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1211 >; 1212 1213 def : GCNPat< 1214 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1215 imm:$cachepolicy, imm), 1216 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1217 (as_i16imm $offset), (extract_glc $cachepolicy), 1218 (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1219 >; 1220 1221 def : GCNPat< 1222 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1223 imm:$cachepolicy, imm), 1224 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1225 $vdata, 1226 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1227 $rsrc, $soffset, (as_i16imm $offset), (extract_glc $cachepolicy), 1228 (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1229 >; 1230} 1231 1232defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1233defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">; 1234defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1235defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">; 1236defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">; 1237defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">; 1238defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1239defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">; 1240 1241let SubtargetPredicate = HasUnpackedD16VMem in { 1242 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1243 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1244 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1245 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1246} // End HasUnpackedD16VMem. 1247 1248let SubtargetPredicate = HasPackedD16VMem in { 1249 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1250 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">; 1251 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1252 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">; 1253 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1254 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1255} // End HasPackedD16VMem. 1256 1257defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1258defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">; 1259defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1260defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">; 1261defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">; 1262defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">; 1263defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1264defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">; 1265defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">; 1266defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">; 1267 1268//===----------------------------------------------------------------------===// 1269// buffer_atomic patterns 1270//===----------------------------------------------------------------------===// 1271 1272multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt, 1273 string opcode> { 1274 def : GCNPat< 1275 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 1276 0, i32:$soffset, imm:$offset, 1277 imm:$cachepolicy, 0)), 1278 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 1279 (as_i16imm $offset), (extract_slc $cachepolicy)) 1280 >; 1281 1282 def : GCNPat< 1283 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1284 0, i32:$soffset, imm:$offset, 1285 imm:$cachepolicy, imm)), 1286 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1287 (as_i16imm $offset), (extract_slc $cachepolicy)) 1288 >; 1289 1290 def : GCNPat< 1291 (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 1292 i32:$voffset, i32:$soffset, imm:$offset, 1293 imm:$cachepolicy, 0)), 1294 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1295 (as_i16imm $offset), (extract_slc $cachepolicy)) 1296 >; 1297 1298 def : GCNPat< 1299 (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1300 i32:$voffset, i32:$soffset, imm:$offset, 1301 imm:$cachepolicy, imm)), 1302 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1303 $vdata_in, 1304 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1305 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1306 >; 1307} 1308 1309defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">; 1310defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">; 1311defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">; 1312defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">; 1313defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">; 1314defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">; 1315defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">; 1316defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">; 1317defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">; 1318defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">; 1319defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">; 1320defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64, "BUFFER_ATOMIC_ADD_X2">; 1321defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">; 1322defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">; 1323defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">; 1324defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">; 1325defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">; 1326defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">; 1327defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">; 1328defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">; 1329 1330multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt, 1331 string opcode> { 1332 def : GCNPat< 1333 (name vt:$vdata_in, v4i32:$rsrc, 0, 1334 0, i32:$soffset, imm:$offset, 1335 imm:$cachepolicy, 0), 1336 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset, 1337 (as_i16imm $offset), (extract_slc $cachepolicy)) 1338 >; 1339 1340 def : GCNPat< 1341 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1342 0, i32:$soffset, imm:$offset, 1343 imm:$cachepolicy, imm), 1344 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset, 1345 (as_i16imm $offset), (extract_slc $cachepolicy)) 1346 >; 1347 1348 def : GCNPat< 1349 (name vt:$vdata_in, v4i32:$rsrc, 0, 1350 i32:$voffset, i32:$soffset, imm:$offset, 1351 imm:$cachepolicy, 0), 1352 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset, 1353 (as_i16imm $offset), (extract_slc $cachepolicy)) 1354 >; 1355 1356 def : GCNPat< 1357 (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 1358 i32:$voffset, i32:$soffset, imm:$offset, 1359 imm:$cachepolicy, imm), 1360 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1361 $vdata_in, 1362 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1363 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)) 1364 >; 1365} 1366 1367defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">; 1368defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">; 1369 1370def : GCNPat< 1371 (SIbuffer_atomic_cmpswap 1372 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1373 0, i32:$soffset, imm:$offset, 1374 imm:$cachepolicy, 0), 1375 (EXTRACT_SUBREG 1376 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1377 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1378 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1379 sub0) 1380>; 1381 1382def : GCNPat< 1383 (SIbuffer_atomic_cmpswap 1384 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1385 0, i32:$soffset, imm:$offset, 1386 imm:$cachepolicy, imm), 1387 (EXTRACT_SUBREG 1388 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1389 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1390 $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1391 sub0) 1392>; 1393 1394def : GCNPat< 1395 (SIbuffer_atomic_cmpswap 1396 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1397 i32:$voffset, i32:$soffset, imm:$offset, 1398 imm:$cachepolicy, 0), 1399 (EXTRACT_SUBREG 1400 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1401 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1402 $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1403 sub0) 1404>; 1405 1406def : GCNPat< 1407 (SIbuffer_atomic_cmpswap 1408 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1409 i32:$voffset, i32:$soffset, imm:$offset, 1410 imm:$cachepolicy, imm), 1411 (EXTRACT_SUBREG 1412 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1413 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1414 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1415 $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)), 1416 sub0) 1417>; 1418 1419class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1420 PatFrag constant_ld> : GCNPat < 1421 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1422 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))), 1423 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc) 1424 >; 1425 1426multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1427 ValueType vt, PatFrag atomic_ld> { 1428 def : GCNPat < 1429 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1430 i16:$offset, i1:$slc))), 1431 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0) 1432 >; 1433 1434 def : GCNPat < 1435 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1436 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0) 1437 >; 1438} 1439 1440let SubtargetPredicate = isGFX6GFX7 in { 1441def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1442def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>; 1443def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>; 1444def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1445def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>; 1446def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>; 1447 1448defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>; 1449defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>; 1450} // End SubtargetPredicate = isGFX6GFX7 1451 1452multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1453 PatFrag ld> { 1454 1455 def : GCNPat < 1456 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1457 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))), 1458 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc) 1459 >; 1460} 1461 1462let OtherPredicates = [Has16BitInsts] in { 1463 1464defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1465defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>; 1466defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>; 1467defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>; 1468defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>; 1469defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>; 1470 1471defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>; 1472 1473} // End OtherPredicates = [Has16BitInsts] 1474 1475multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1476 MUBUF_Pseudo InstrOffset, 1477 ValueType vt, PatFrag ld> { 1478 def : GCNPat < 1479 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1480 i32:$soffset, u16imm:$offset))), 1481 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0) 1482 >; 1483 1484 def : GCNPat < 1485 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1486 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0) 1487 >; 1488} 1489 1490// XXX - Is it possible to have a complex pattern in a PatFrag? 1491multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen, 1492 MUBUF_Pseudo InstrOffset, 1493 ValueType vt, PatFrag ld_frag> { 1494 def : GCNPat < 1495 (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), 1496 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, $in) 1497 >; 1498 1499 def : GCNPat < 1500 (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), 1501 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, $in) 1502 >; 1503} 1504 1505defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1506defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>; 1507defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>; 1508defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1509defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>; 1510defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>; 1511defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1512defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>; 1513defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>; 1514defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1515defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1516defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1517defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>; 1518defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1519 1520let OtherPredicates = [D16PreservesUnusedBits] in { 1521defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>; 1522defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>; 1523defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>; 1524defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>; 1525defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>; 1526defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>; 1527 1528defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>; 1529defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>; 1530defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>; 1531defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>; 1532defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>; 1533defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>; 1534} 1535 1536multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1537 ValueType vt, PatFrag atomic_st> { 1538 // Store follows atomic op convention so address is forst 1539 def : GCNPat < 1540 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1541 i16:$offset, i1:$slc), vt:$val), 1542 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0) 1543 >; 1544 1545 def : GCNPat < 1546 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1547 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0) 1548 >; 1549} 1550let SubtargetPredicate = isGFX6GFX7 in { 1551defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1552defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1553} // End Predicates = isGFX6GFX7 1554 1555 1556multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1557 PatFrag st> { 1558 1559 def : GCNPat < 1560 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1561 i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)), 1562 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc) 1563 >; 1564} 1565 1566defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1567defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1568 1569multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1570 MUBUF_Pseudo InstrOffset, 1571 ValueType vt, PatFrag st, 1572 RegisterClass rc = VGPR_32> { 1573 def : GCNPat < 1574 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1575 i32:$soffset, u16imm:$offset)), 1576 (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0) 1577 >; 1578 1579 def : GCNPat < 1580 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1581 u16imm:$offset)), 1582 (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0) 1583 >; 1584} 1585 1586defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1587defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1588defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1589defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1590defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1591defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>; 1592defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>; 1593defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>; 1594 1595 1596let OtherPredicates = [D16PreservesUnusedBits] in { 1597 // Hiding the extract high pattern in the PatFrag seems to not 1598 // automatically increase the complexity. 1599let AddedComplexity = 1 in { 1600defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1601defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1602} 1603} 1604 1605//===----------------------------------------------------------------------===// 1606// MTBUF Patterns 1607//===----------------------------------------------------------------------===// 1608 1609//===----------------------------------------------------------------------===// 1610// tbuffer_load/store_format patterns 1611//===----------------------------------------------------------------------===// 1612 1613multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1614 string opcode> { 1615 def : GCNPat< 1616 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1617 imm:$format, imm:$cachepolicy, 0)), 1618 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1619 (as_i8imm $format), 1620 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1621 >; 1622 1623 def : GCNPat< 1624 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1625 imm:$format, imm:$cachepolicy, imm)), 1626 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1627 (as_i8imm $format), 1628 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1629 >; 1630 1631 def : GCNPat< 1632 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1633 imm:$format, imm:$cachepolicy, 0)), 1634 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1635 (as_i8imm $format), 1636 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1637 >; 1638 1639 def : GCNPat< 1640 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1641 imm:$format, imm:$cachepolicy, imm)), 1642 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1643 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1644 $rsrc, $soffset, (as_i16imm $offset), 1645 (as_i8imm $format), 1646 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1647 >; 1648} 1649 1650defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1651defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1652defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">; 1653defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1654defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1655defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1656defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">; 1657defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1658 1659let SubtargetPredicate = HasUnpackedD16VMem in { 1660 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1661 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1662 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1663} // End HasUnpackedD16VMem. 1664 1665let SubtargetPredicate = HasPackedD16VMem in { 1666 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1667 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1668 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1669} // End HasPackedD16VMem. 1670 1671multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1672 string opcode> { 1673 def : GCNPat< 1674 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1675 imm:$format, imm:$cachepolicy, 0), 1676 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1677 (as_i16imm $offset), (as_i8imm $format), 1678 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1679 >; 1680 1681 def : GCNPat< 1682 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1683 imm:$format, imm:$cachepolicy, imm), 1684 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1685 (as_i16imm $offset), (as_i8imm $format), 1686 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1687 >; 1688 1689 def : GCNPat< 1690 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1691 imm:$format, imm:$cachepolicy, 0), 1692 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1693 (as_i16imm $offset), (as_i8imm $format), 1694 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1695 >; 1696 1697 def : GCNPat< 1698 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1699 imm:$offset, imm:$format, imm:$cachepolicy, imm), 1700 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1701 $vdata, 1702 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1703 $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format), 1704 (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy)) 1705 >; 1706} 1707 1708defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1709defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1710defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">; 1711defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1712defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1713defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1714defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">; 1715defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1716 1717let SubtargetPredicate = HasUnpackedD16VMem in { 1718 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1719 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1720 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1721} // End HasUnpackedD16VMem. 1722 1723let SubtargetPredicate = HasPackedD16VMem in { 1724 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1725 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1726 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1727} // End HasPackedD16VMem. 1728 1729//===----------------------------------------------------------------------===// 1730// Target-specific instruction encodings. 1731//===----------------------------------------------------------------------===// 1732 1733//===----------------------------------------------------------------------===// 1734// Base ENC_MUBUF for GFX6, GFX7, GFX10. 1735//===----------------------------------------------------------------------===// 1736 1737class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> : 1738 MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1739 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1740 let Inst{12} = ps.offen; 1741 let Inst{13} = ps.idxen; 1742 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1743 let Inst{16} = !if(ps.lds, 1, 0); 1744 let Inst{24-18} = op; 1745 let Inst{31-26} = 0x38; 1746 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1747 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1748 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1749 let Inst{54} = !if(ps.has_slc, slc, ?); 1750 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1751 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1752} 1753 1754class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> : 1755 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> { 1756 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 1757 let Inst{25} = op{7}; 1758} 1759 1760class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> : 1761 Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> { 1762 let Inst{15} = ps.addr64; 1763} 1764 1765//===----------------------------------------------------------------------===// 1766// MUBUF - GFX10. 1767//===----------------------------------------------------------------------===// 1768 1769let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 1770 multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName, 1771 string asmName> { 1772 def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> { 1773 MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName); 1774 let AsmString = asmName # ps.AsmOperands; 1775 } 1776 } 1777 multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> { 1778 def _BOTHEN_gfx10 : 1779 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1780 def _IDXEN_gfx10 : 1781 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1782 def _OFFEN_gfx10 : 1783 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1784 def _OFFSET_gfx10 : 1785 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1786 } 1787 multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> { 1788 def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1789 MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">; 1790 def _OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1791 MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">; 1792 def _IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1793 MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">; 1794 def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1795 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">; 1796 1797 def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1798 MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">; 1799 def _LDS_OFFEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1800 MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">; 1801 def _LDS_IDXEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1802 MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">; 1803 def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1804 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">; 1805 } 1806 multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> : 1807 MUBUF_Real_AllAddr_gfx10<op> { 1808 def _BOTHEN_RTN_gfx10 : 1809 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1810 def _IDXEN_RTN_gfx10 : 1811 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1812 def _OFFEN_RTN_gfx10 : 1813 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1814 def _OFFSET_RTN_gfx10 : 1815 MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1816 } 1817} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 1818 1819defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x019>; 1820defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x01b>; 1821defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x020>; 1822defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x021>; 1823defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_gfx10<0x022>; 1824defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_gfx10<0x023>; 1825defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_gfx10<0x024>; 1826defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_gfx10<0x025>; 1827// FIXME-GFX10: Add following instructions: 1828//defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x026>; 1829//defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>; 1830defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x080>; 1831defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x081>; 1832defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x082>; 1833defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x083>; 1834defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_gfx10<0x084>; 1835defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_gfx10<0x085>; 1836defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_gfx10<0x086>; 1837defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>; 1838 1839def BUFFER_GL0_INV_gfx10 : 1840 MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>; 1841def BUFFER_GL1_INV_gfx10 : 1842 MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>; 1843 1844//===----------------------------------------------------------------------===// 1845// MUBUF - GFX6, GFX7, GFX10. 1846//===----------------------------------------------------------------------===// 1847 1848let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in { 1849 multiclass MUBUF_Real_gfx6<bits<8> op> { 1850 def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1851 } 1852} // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" 1853 1854let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in { 1855 multiclass MUBUF_Real_gfx7<bits<8> op> { 1856 def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>; 1857 } 1858} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" 1859 1860let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 1861 multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> { 1862 def _ADDR64_gfx6_gfx7 : 1863 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1864 def _BOTHEN_gfx6_gfx7 : 1865 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1866 def _IDXEN_gfx6_gfx7 : 1867 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1868 def _OFFEN_gfx6_gfx7 : 1869 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1870 def _OFFSET_gfx6_gfx7 : 1871 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1872 } 1873 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> { 1874 def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1875 MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">; 1876 def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1877 MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">; 1878 def _OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1879 MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">; 1880 def _IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1881 MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">; 1882 def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1883 MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">; 1884 1885 def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1886 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">; 1887 def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1888 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">; 1889 def _LDS_OFFEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1890 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">; 1891 def _LDS_IDXEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1892 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">; 1893 def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1894 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">; 1895 } 1896 multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> : 1897 MUBUF_Real_AllAddr_gfx6_gfx7<op> { 1898 def _ADDR64_RTN_gfx6_gfx7 : 1899 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1900 def _BOTHEN_RTN_gfx6_gfx7 : 1901 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1902 def _IDXEN_RTN_gfx6_gfx7 : 1903 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1904 def _OFFEN_RTN_gfx6_gfx7 : 1905 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1906 def _OFFSET_RTN_gfx6_gfx7 : 1907 MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1908 } 1909} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 1910 1911multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> : 1912 MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>; 1913 1914multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> : 1915 MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>; 1916 1917multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> : 1918 MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>; 1919 1920// FIXME-GFX6: Following instructions are available only on GFX6. 1921//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomics_gfx6 <0x034>; 1922//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomics_gfx6 <0x054>; 1923 1924defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>; 1925defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 1926defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 1927defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 1928defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 1929defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 1930defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 1931defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 1932defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>; 1933defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>; 1934defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>; 1935defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>; 1936defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>; 1937defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>; 1938defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>; 1939defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>; 1940defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>; 1941defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>; 1942defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>; 1943defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>; 1944defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>; 1945defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>; 1946 1947defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>; 1948defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>; 1949defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>; 1950defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>; 1951defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>; 1952defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>; 1953defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>; 1954defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>; 1955defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>; 1956defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>; 1957defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>; 1958defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>; 1959defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>; 1960// FIXME-GFX6-GFX7-GFX10: Add following instructions: 1961//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>; 1962//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>; 1963//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>; 1964defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>; 1965defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>; 1966defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>; 1967defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>; 1968defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>; 1969defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>; 1970defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>; 1971defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>; 1972defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>; 1973defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>; 1974defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>; 1975defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>; 1976defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>; 1977// FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7. 1978// FIXME-GFX6-GFX7-GFX10: Add following instructions: 1979//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>; 1980//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>; 1981//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>; 1982 1983defm BUFFER_WBINVL1_SC : MUBUF_Real_gfx6<0x070>; 1984defm BUFFER_WBINVL1_VOL : MUBUF_Real_gfx7<0x070>; 1985def BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>; 1986 1987//===----------------------------------------------------------------------===// 1988// Base ENC_MTBUF for GFX6, GFX7, GFX10. 1989//===----------------------------------------------------------------------===// 1990 1991class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> : 1992 MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> { 1993 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1994 let Inst{12} = ps.offen; 1995 let Inst{13} = ps.idxen; 1996 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1997 let Inst{18-16} = op; 1998 let Inst{31-26} = 0x3a; //encoding 1999 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2000 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2001 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2002 let Inst{54} = !if(ps.has_slc, slc, ?); 2003 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2004 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2005} 2006 2007//===----------------------------------------------------------------------===// 2008// MTBUF - GFX10. 2009//===----------------------------------------------------------------------===// 2010 2011class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> : 2012 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> { 2013 let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value); 2014 let Inst{25-19} = format; 2015 let Inst{53} = op{3}; 2016} 2017 2018let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { 2019 multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> { 2020 def _BOTHEN_gfx10 : 2021 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2022 def _IDXEN_gfx10 : 2023 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2024 def _OFFEN_gfx10 : 2025 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2026 def _OFFSET_gfx10 : 2027 MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2028 } 2029} // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" 2030 2031defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x008>; 2032defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x009>; 2033defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00a>; 2034defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00b>; 2035defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_gfx10<0x00c>; 2036defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_gfx10<0x00d>; 2037defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_gfx10<0x00e>; 2038defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>; 2039 2040//===----------------------------------------------------------------------===// 2041// MTBUF - GFX6, GFX7, GFX10. 2042//===----------------------------------------------------------------------===// 2043 2044class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> : 2045 Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> { 2046 let Inst{15} = ps.addr64; 2047 let Inst{22-19} = dfmt; 2048 let Inst{25-23} = nfmt; 2049} 2050 2051let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in { 2052 multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> { 2053 def _ADDR64_gfx6_gfx7 : 2054 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 2055 def _BOTHEN_gfx6_gfx7 : 2056 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2057 def _IDXEN_gfx6_gfx7 : 2058 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2059 def _OFFEN_gfx6_gfx7 : 2060 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2061 def _OFFSET_gfx6_gfx7 : 2062 MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2063 } 2064} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" 2065 2066multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> : 2067 MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>; 2068 2069defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>; 2070defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>; 2071defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>; 2072defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>; 2073defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>; 2074defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>; 2075defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>; 2076defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>; 2077 2078//===----------------------------------------------------------------------===// 2079// GFX8, GFX9 (VI). 2080//===----------------------------------------------------------------------===// 2081 2082class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 2083 MUBUF_Real<ps>, 2084 Enc64, 2085 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2086 let AssemblerPredicate = isGFX8GFX9; 2087 let DecoderNamespace = "GFX8"; 2088 2089 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2090 let Inst{12} = ps.offen; 2091 let Inst{13} = ps.idxen; 2092 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2093 let Inst{16} = !if(ps.lds, 1, 0); 2094 let Inst{17} = !if(ps.has_slc, slc, ?); 2095 let Inst{24-18} = op; 2096 let Inst{31-26} = 0x38; //encoding 2097 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2098 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2099 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2100 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2101 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2102} 2103 2104multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 2105 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2106 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2107 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2108 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2109} 2110 2111multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 2112 2113 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 2114 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 2115 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 2116 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 2117 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 2118 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 2119 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 2120 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 2121 2122 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 2123 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 2124 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 2125 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 2126 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 2127 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 2128 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 2129 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 2130} 2131 2132class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 2133 MUBUF_Real<ps>, 2134 Enc64, 2135 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2136 let AssemblerPredicate=HasUnpackedD16VMem; 2137 let DecoderNamespace="GFX80_UNPACKED"; 2138 2139 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2140 let Inst{12} = ps.offen; 2141 let Inst{13} = ps.idxen; 2142 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2143 let Inst{16} = !if(ps.lds, 1, 0); 2144 let Inst{17} = !if(ps.has_slc, slc, ?); 2145 let Inst{24-18} = op; 2146 let Inst{31-26} = 0x38; //encoding 2147 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2148 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2149 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2150 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2151 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2152} 2153 2154multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 2155 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 2156 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 2157 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 2158 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 2159} 2160 2161multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 2162 MUBUF_Real_AllAddr_vi<op> { 2163 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 2164 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 2165 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 2166 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 2167} 2168 2169defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 2170defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 2171defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 2172defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 2173defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 2174defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 2175defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 2176defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 2177let SubtargetPredicate = HasUnpackedD16VMem in { 2178 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 2179 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 2180 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 2181 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 2182 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 2183 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 2184 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 2185 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 2186} // End HasUnpackedD16VMem. 2187let SubtargetPredicate = HasPackedD16VMem in { 2188 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 2189 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 2190 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 2191 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 2192 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 2193 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 2194 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 2195 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 2196} // End HasPackedD16VMem. 2197defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 2198defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 2199defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 2200defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 2201defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 2202defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 2203defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 2204defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 2205defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 2206defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 2207defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 2208defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 2209defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 2210defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 2211defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 2212defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 2213 2214defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 2215defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 2216defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 2217defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 2218defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 2219defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 2220 2221defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 2222defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 2223 2224defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 2225defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 2226defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 2227defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 2228defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 2229defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 2230defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 2231defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 2232defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 2233defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 2234defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 2235defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 2236defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 2237 2238defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 2239defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 2240defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 2241defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 2242defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 2243defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 2244defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 2245defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 2246defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 2247defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 2248defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 2249defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 2250defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 2251 2252def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2253 2254def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2255def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2256 2257let SubtargetPredicate = HasAtomicFaddInsts in { 2258 2259defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Real_AllAddr_vi <0x4d>; 2260defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>; 2261 2262} // End SubtargetPredicate = HasAtomicFaddInsts 2263 2264class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2265 MTBUF_Real<ps>, 2266 Enc64, 2267 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2268 let AssemblerPredicate = isGFX8GFX9; 2269 let DecoderNamespace = "GFX8"; 2270 2271 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2272 let Inst{12} = ps.offen; 2273 let Inst{13} = ps.idxen; 2274 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2275 let Inst{18-15} = op; 2276 let Inst{22-19} = dfmt; 2277 let Inst{25-23} = nfmt; 2278 let Inst{31-26} = 0x3a; //encoding 2279 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2280 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2281 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2282 let Inst{54} = !if(ps.has_slc, slc, ?); 2283 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2284 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2285} 2286 2287multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2288 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2289 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2290 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2291 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2292} 2293 2294class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2295 MTBUF_Real<ps>, 2296 Enc64, 2297 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2298 let AssemblerPredicate=HasUnpackedD16VMem; 2299 let DecoderNamespace="GFX80_UNPACKED"; 2300 2301 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2302 let Inst{12} = ps.offen; 2303 let Inst{13} = ps.idxen; 2304 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2305 let Inst{18-15} = op; 2306 let Inst{22-19} = dfmt; 2307 let Inst{25-23} = nfmt; 2308 let Inst{31-26} = 0x3a; //encoding 2309 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2310 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2311 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2312 let Inst{54} = !if(ps.has_slc, slc, ?); 2313 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2314 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2315} 2316 2317multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2318 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2319 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2320 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2321 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2322} 2323 2324defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2325defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2326defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2327defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2328defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2329defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2330defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2331defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2332let SubtargetPredicate = HasUnpackedD16VMem in { 2333 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2334 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2335 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2336 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2337 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2338 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2339 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2340 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2341} // End HasUnpackedD16VMem. 2342let SubtargetPredicate = HasPackedD16VMem in { 2343 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2344 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2345 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2346 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2347 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2348 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2349 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2350 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2351} // End HasUnpackedD16VMem. 2352 2353def MUBUFInfoTable : GenericTable { 2354 let FilterClass = "MUBUF_Pseudo"; 2355 let CppTypeName = "MUBUFInfo"; 2356 let Fields = ["Opcode", "BaseOpcode", "dwords", "has_vaddr", "has_srsrc", "has_soffset"]; 2357 2358 let PrimaryKey = ["Opcode"]; 2359 let PrimaryKeyName = "getMUBUFOpcodeHelper"; 2360} 2361 2362def getMUBUFInfoFromOpcode : SearchIndex { 2363 let Table = MUBUFInfoTable; 2364 let Key = ["Opcode"]; 2365} 2366 2367def getMUBUFInfoFromBaseOpcodeAndDwords : SearchIndex { 2368 let Table = MUBUFInfoTable; 2369 let Key = ["BaseOpcode", "dwords"]; 2370} 2371