1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8 9def smrd_offset_8 : ImmOperand<i32, "SMRDOffset8", 1>; 10 11let EncoderMethod = "getSMEMOffsetEncoding", 12 DecoderMethod = "decodeSMEMOffset" in { 13def smem_offset : ImmOperand<i32, "SMEMOffset", 1>; 14def smem_offset_mod : NamedIntOperand<i32, "offset", "SMEMOffsetMod">; 15} 16 17//===----------------------------------------------------------------------===// 18// Scalar Memory classes 19//===----------------------------------------------------------------------===// 20 21class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> : 22 InstSI <outs, ins, "", pattern>, 23 SIMCInstr<opName, SIEncodingFamily.NONE> { 24 let isPseudo = 1; 25 let isCodeGenOnly = 1; 26 27 let LGKM_CNT = 1; 28 let SMRD = 1; 29 let mayStore = 0; 30 let mayLoad = 1; 31 let hasSideEffects = 0; 32 let UseNamedOperandTable = 1; 33 let SchedRW = [WriteSMEM]; 34 35 string Mnemonic = opName; 36 string AsmOperands = asmOps; 37 38 bits<1> has_sbase = 1; 39 bits<1> has_sdst = 1; 40 bit has_glc = 0; 41 bit has_dlc = 0; 42 bit has_offset = 0; 43 bit has_soffset = 0; 44 bit is_buffer = 0; 45} 46 47class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic> 48 : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> { 49 50 let isPseudo = 0; 51 let isCodeGenOnly = 0; 52 53 Instruction Opcode = !cast<Instruction>(NAME); 54 55 // copy relevant pseudo op flags 56 let LGKM_CNT = ps.LGKM_CNT; 57 let SMRD = ps.SMRD; 58 let mayStore = ps.mayStore; 59 let mayLoad = ps.mayLoad; 60 let hasSideEffects = ps.hasSideEffects; 61 let UseNamedOperandTable = ps.UseNamedOperandTable; 62 let SchedRW = ps.SchedRW; 63 let SubtargetPredicate = ps.SubtargetPredicate; 64 let AsmMatchConverter = ps.AsmMatchConverter; 65 let IsAtomicRet = ps.IsAtomicRet; 66 let IsAtomicNoRet = ps.IsAtomicNoRet; 67 68 let TSFlags = ps.TSFlags; 69 70 bit is_buffer = ps.is_buffer; 71 72 // encoding 73 bits<7> sbase; 74 bits<7> sdst; 75 bits<32> offset; 76 bits<8> soffset; 77 bits<5> cpol; 78} 79 80class OffsetMode<bit hasOffset, bit hasSOffset, string variant, 81 dag ins, string asm> { 82 bit HasOffset = hasOffset; 83 bit HasSOffset = hasSOffset; 84 string Variant = variant; 85 dag Ins = ins; 86 string Asm = asm; 87} 88 89def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins smem_offset:$offset), "$offset">; 90def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">; 91def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM", 92 (ins SReg_32:$soffset, smem_offset_mod:$offset), 93 "$soffset$offset">; 94 95class SM_Probe_Pseudo <string opName, RegisterClass baseClass, OffsetMode offsets> 96 : SM_Pseudo<opName, (outs), 97 !con((ins i8imm:$sdata, baseClass:$sbase), offsets.Ins), 98 " $sdata, $sbase, " # offsets.Asm> { 99 let mayLoad = 0; 100 let mayStore = 0; 101 let has_glc = 0; 102 let LGKM_CNT = 0; 103 let ScalarStore = 0; 104 let hasSideEffects = 1; 105 let has_offset = offsets.HasOffset; 106 let has_soffset = offsets.HasSOffset; 107 let PseudoInstr = opName # offsets.Variant; 108} 109 110class SM_Load_Pseudo <string opName, RegisterClass baseClass, 111 RegisterClass dstClass, OffsetMode offsets> 112 : SM_Pseudo<opName, (outs dstClass:$sdst), 113 !con((ins baseClass:$sbase), offsets.Ins, (ins CPol:$cpol)), 114 " $sdst, $sbase, " # offsets.Asm # "$cpol", []> { 115 RegisterClass BaseClass = baseClass; 116 let mayLoad = 1; 117 let isReMaterializable = 1; 118 let mayStore = 0; 119 let has_glc = 1; 120 let has_dlc = 1; 121 let has_offset = offsets.HasOffset; 122 let has_soffset = offsets.HasSOffset; 123 let PseudoInstr = opName # offsets.Variant; 124} 125 126class SM_Store_Pseudo <string opName, RegisterClass baseClass, 127 RegisterClass srcClass, OffsetMode offsets> 128 : SM_Pseudo<opName, (outs), !con((ins srcClass:$sdata, baseClass:$sbase), 129 offsets.Ins, (ins CPol:$cpol)), 130 " $sdata, $sbase, " # offsets.Asm # "$cpol"> { 131 RegisterClass BaseClass = baseClass; 132 let mayLoad = 0; 133 let mayStore = 1; 134 let has_glc = 1; 135 let has_dlc = 1; 136 let has_offset = offsets.HasOffset; 137 let has_soffset = offsets.HasSOffset; 138 let ScalarStore = 1; 139 let PseudoInstr = opName # offsets.Variant; 140} 141 142class SM_Discard_Pseudo <string opName, OffsetMode offsets> 143 : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets.Ins), 144 " $sbase, " # offsets.Asm> { 145 let mayLoad = 0; 146 let mayStore = 0; 147 let has_glc = 0; 148 let has_sdst = 0; 149 let ScalarStore = 0; 150 let hasSideEffects = 1; 151 let has_offset = offsets.HasOffset; 152 let has_soffset = offsets.HasSOffset; 153 let PseudoInstr = opName # offsets.Variant; 154} 155 156multiclass SM_Pseudo_Loads<RegisterClass baseClass, 157 RegisterClass dstClass> { 158 defvar opName = !tolower(NAME); 159 def _IMM : SM_Load_Pseudo <opName, baseClass, dstClass, IMM_Offset>; 160 def _SGPR : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_Offset>; 161 def _SGPR_IMM : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_IMM_Offset>; 162} 163 164multiclass SM_Pseudo_Stores<RegisterClass baseClass, 165 RegisterClass srcClass> { 166 defvar opName = !tolower(NAME); 167 def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass, IMM_Offset>; 168 def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_Offset>; 169 def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass, SGPR_IMM_Offset>; 170} 171 172multiclass SM_Pseudo_Discards { 173 defvar opName = !tolower(NAME); 174 def _IMM : SM_Discard_Pseudo <opName, IMM_Offset>; 175 def _SGPR : SM_Discard_Pseudo <opName, SGPR_Offset>; 176 def _SGPR_IMM : SM_Discard_Pseudo <opName, SGPR_IMM_Offset>; 177} 178 179class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 180 opName, (outs SReg_64_XEXEC:$sdst), (ins), 181 " $sdst", [(set i64:$sdst, (node))]> { 182 let hasSideEffects = 1; 183 184 let mayStore = 0; 185 let mayLoad = 0; 186 let has_sbase = 0; 187} 188 189class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo< 190 opName, (outs), (ins), "", [(node)]> { 191 let hasSideEffects = 1; 192 let mayLoad = 0; 193 let mayStore = 0; 194 let has_sdst = 0; 195 let has_sbase = 0; 196} 197 198multiclass SM_Pseudo_Probe<RegisterClass baseClass> { 199 defvar opName = !tolower(NAME); 200 def _IMM : SM_Probe_Pseudo <opName, baseClass, IMM_Offset>; 201 def _SGPR : SM_Probe_Pseudo <opName, baseClass, SGPR_Offset>; 202 def _SGPR_IMM : SM_Probe_Pseudo <opName, baseClass, SGPR_IMM_Offset>; 203} 204 205class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo< 206 opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins), 207 " $sdst", [(set i32:$sdst, (node))]> { 208 let hasSideEffects = 1; 209 let mayStore = 0; 210 let mayLoad = 0; 211 let has_sbase = 0; 212} 213 214//===----------------------------------------------------------------------===// 215// Scalar Atomic Memory Classes 216//===----------------------------------------------------------------------===// 217 218class SM_Atomic_Pseudo <string opName, 219 dag outs, dag ins, string asmOps, bit isRet> 220 : SM_Pseudo<opName, outs, ins, asmOps, []> { 221 222 bit glc = isRet; 223 224 let mayLoad = 1; 225 let mayStore = 1; 226 let has_glc = 1; 227 let has_dlc = 1; 228 let has_soffset = 1; 229 230 // Should these be set? 231 let ScalarStore = 1; 232 let hasSideEffects = 1; 233 let maybeAtomic = 1; 234 235 let IsAtomicNoRet = !not(isRet); 236 let IsAtomicRet = isRet; 237 238 let AsmMatchConverter = "cvtSMEMAtomic"; 239} 240 241class SM_Pseudo_Atomic<string opName, 242 RegisterClass baseClass, 243 RegisterClass dataClass, 244 OffsetMode offsets, 245 bit isRet, 246 string opNameWithSuffix = 247 opName # offsets.Variant # !if(isRet, "_RTN", ""), 248 Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> : 249 SM_Atomic_Pseudo<opName, 250 !if(isRet, (outs dataClass:$sdst), (outs)), 251 !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins, 252 (ins CPolTy:$cpol)), 253 !if(isRet, " $sdst", " $sdata") # 254 ", $sbase, " # offsets.Asm # "$cpol", 255 isRet>, 256 AtomicNoRet <opNameWithSuffix, isRet> { 257 let has_offset = offsets.HasOffset; 258 let has_soffset = offsets.HasSOffset; 259 let PseudoInstr = opNameWithSuffix; 260 261 let Constraints = !if(isRet, "$sdst = $sdata", ""); 262 let DisableEncoding = !if(isRet, "$sdata", ""); 263} 264 265multiclass SM_Pseudo_Atomics<RegisterClass baseClass, 266 RegisterClass dataClass> { 267 defvar opName = !tolower(NAME); 268 def _IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>; 269 def _SGPR : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>; 270 def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>; 271 def _IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>; 272 def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>; 273 def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>; 274} 275 276//===----------------------------------------------------------------------===// 277// Scalar Memory Instructions 278//===----------------------------------------------------------------------===// 279 280// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit 281// SMRD instructions, because the SReg_32_XM0 register class does not include M0 282// and writing to M0 from an SMRD instruction will hang the GPU. 283 284// XXX - SMEM instructions do not allow exec for data operand, but 285// does sdst for SMRD on SI/CI? 286defm S_LOAD_DWORD : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 287defm S_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>; 288defm S_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_64, SReg_128>; 289defm S_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_64, SReg_256>; 290defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_64, SReg_512>; 291 292let is_buffer = 1 in { 293defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <SReg_128, SReg_32_XM0_XEXEC>; 294// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on 295// SI/CI, bit disallowed for SMEM on VI. 296defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_128, SReg_64_XEXEC>; 297defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_128, SReg_128>; 298defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <SReg_128, SReg_256>; 299defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <SReg_128, SReg_512>; 300} 301 302let SubtargetPredicate = HasScalarStores in { 303defm S_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>; 304defm S_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>; 305defm S_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>; 306 307let is_buffer = 1 in { 308defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <SReg_128, SReg_32_XM0_XEXEC>; 309defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_128, SReg_64_XEXEC>; 310defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_128, SReg_128>; 311} 312} // End SubtargetPredicate = HasScalarStores 313 314let SubtargetPredicate = HasSMemTimeInst in 315def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>; 316def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>; 317 318let SubtargetPredicate = isGFX7GFX8GFX9 in { 319def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>; 320} // let SubtargetPredicate = isGFX7GFX8GFX9 321 322let SubtargetPredicate = isGFX8Plus in { 323let OtherPredicates = [HasScalarStores] in { 324def S_DCACHE_WB : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>; 325def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>; 326} // End OtherPredicates = [HasScalarStores] 327 328defm S_ATC_PROBE : SM_Pseudo_Probe <SReg_64>; 329let is_buffer = 1 in { 330defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <SReg_128>; 331} 332} // SubtargetPredicate = isGFX8Plus 333 334let SubtargetPredicate = HasSMemRealTime in 335def S_MEMREALTIME : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>; 336 337let SubtargetPredicate = isGFX10Plus in 338def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">; 339let SubtargetPredicate = HasGetWaveIdInst in 340def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>; 341 342 343let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in { 344defm S_SCRATCH_LOAD_DWORD : SM_Pseudo_Loads <SReg_64, SReg_32_XM0_XEXEC>; 345defm S_SCRATCH_LOAD_DWORDX2 : SM_Pseudo_Loads <SReg_64, SReg_64_XEXEC>; 346defm S_SCRATCH_LOAD_DWORDX4 : SM_Pseudo_Loads <SReg_64, SReg_128>; 347 348defm S_SCRATCH_STORE_DWORD : SM_Pseudo_Stores <SReg_64, SReg_32_XM0_XEXEC>; 349defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <SReg_64, SReg_64_XEXEC>; 350defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <SReg_64, SReg_128>; 351} // SubtargetPredicate = HasScalarFlatScratchInsts 352 353let SubtargetPredicate = HasScalarAtomics in { 354 355let is_buffer = 1 in { 356defm S_BUFFER_ATOMIC_SWAP : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 357defm S_BUFFER_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 358defm S_BUFFER_ATOMIC_ADD : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 359defm S_BUFFER_ATOMIC_SUB : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 360defm S_BUFFER_ATOMIC_SMIN : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 361defm S_BUFFER_ATOMIC_UMIN : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 362defm S_BUFFER_ATOMIC_SMAX : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 363defm S_BUFFER_ATOMIC_UMAX : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 364defm S_BUFFER_ATOMIC_AND : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 365defm S_BUFFER_ATOMIC_OR : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 366defm S_BUFFER_ATOMIC_XOR : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 367defm S_BUFFER_ATOMIC_INC : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 368defm S_BUFFER_ATOMIC_DEC : SM_Pseudo_Atomics <SReg_128, SReg_32_XM0_XEXEC>; 369 370defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 371defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <SReg_128, SReg_128>; 372defm S_BUFFER_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 373defm S_BUFFER_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 374defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 375defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 376defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 377defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 378defm S_BUFFER_ATOMIC_AND_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 379defm S_BUFFER_ATOMIC_OR_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 380defm S_BUFFER_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 381defm S_BUFFER_ATOMIC_INC_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 382defm S_BUFFER_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <SReg_128, SReg_64_XEXEC>; 383} 384 385defm S_ATOMIC_SWAP : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 386defm S_ATOMIC_CMPSWAP : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 387defm S_ATOMIC_ADD : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 388defm S_ATOMIC_SUB : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 389defm S_ATOMIC_SMIN : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 390defm S_ATOMIC_UMIN : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 391defm S_ATOMIC_SMAX : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 392defm S_ATOMIC_UMAX : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 393defm S_ATOMIC_AND : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 394defm S_ATOMIC_OR : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 395defm S_ATOMIC_XOR : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 396defm S_ATOMIC_INC : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 397defm S_ATOMIC_DEC : SM_Pseudo_Atomics <SReg_64, SReg_32_XM0_XEXEC>; 398 399defm S_ATOMIC_SWAP_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 400defm S_ATOMIC_CMPSWAP_X2 : SM_Pseudo_Atomics <SReg_64, SReg_128>; 401defm S_ATOMIC_ADD_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 402defm S_ATOMIC_SUB_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 403defm S_ATOMIC_SMIN_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 404defm S_ATOMIC_UMIN_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 405defm S_ATOMIC_SMAX_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 406defm S_ATOMIC_UMAX_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 407defm S_ATOMIC_AND_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 408defm S_ATOMIC_OR_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 409defm S_ATOMIC_XOR_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 410defm S_ATOMIC_INC_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 411defm S_ATOMIC_DEC_X2 : SM_Pseudo_Atomics <SReg_64, SReg_64_XEXEC>; 412 413} // let SubtargetPredicate = HasScalarAtomics 414 415let SubtargetPredicate = HasScalarAtomics in { 416defm S_DCACHE_DISCARD : SM_Pseudo_Discards; 417defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards; 418} 419 420//===----------------------------------------------------------------------===// 421// Targets 422//===----------------------------------------------------------------------===// 423 424//===----------------------------------------------------------------------===// 425// SI 426//===----------------------------------------------------------------------===// 427 428class SMRD_Real_si <bits<5> op, SM_Pseudo ps> 429 : SM_Real<ps> 430 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 431 , Enc32 { 432 433 let AssemblerPredicate = isGFX6GFX7; 434 let DecoderNamespace = "GFX6GFX7"; 435 436 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 437 let Inst{8} = ps.has_offset; 438 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 439 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 440 let Inst{26-22} = op; 441 let Inst{31-27} = 0x18; //encoding 442} 443 444multiclass SM_Real_Loads_si<bits<5> op> { 445 defvar ps = NAME; 446 defvar immPs = !cast<SM_Load_Pseudo>(ps#_IMM); 447 def _IMM_si : SMRD_Real_si <op, immPs> { 448 let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol); 449 } 450 451 defvar sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR); 452 def _SGPR_si : SMRD_Real_si <op, sgprPs>; 453} 454 455defm S_LOAD_DWORD : SM_Real_Loads_si <0x00>; 456defm S_LOAD_DWORDX2 : SM_Real_Loads_si <0x01>; 457defm S_LOAD_DWORDX4 : SM_Real_Loads_si <0x02>; 458defm S_LOAD_DWORDX8 : SM_Real_Loads_si <0x03>; 459defm S_LOAD_DWORDX16 : SM_Real_Loads_si <0x04>; 460defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_si <0x08>; 461defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_si <0x09>; 462defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_si <0x0a>; 463defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_si <0x0b>; 464defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c>; 465 466def S_MEMTIME_si : SMRD_Real_si <0x1e, S_MEMTIME>; 467def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>; 468 469 470//===----------------------------------------------------------------------===// 471// VI and GFX9. 472//===----------------------------------------------------------------------===// 473 474class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> 475 : SM_Real<ps> 476 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> 477 , Enc64 { 478 field bit IsGFX9SpecificEncoding = false; 479 let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9); 480 let DecoderNamespace = "GFX8"; 481 482 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 483 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 484 485 // Note that for GFX9 instructions with immediate offsets, soffset_en 486 // must be defined, whereas in GFX8 it's undefined in all cases, 487 // meaning GFX9 is not perfectly backward-compatible with GFX8, despite 488 // documentation suggesting otherwise. 489 field bit SOffsetEn = !if(IsGFX9SpecificEncoding, 490 !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)), 491 ?); 492 let Inst{14} = SOffsetEn; 493 494 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 495 496 // imm 497 // TODO: Shall not be defined if the instruction has no offset nor 498 // soffset. 499 let Inst{17} = ps.has_offset; 500 501 let Inst{25-18} = op; 502 let Inst{31-26} = 0x30; //encoding 503 504 // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed. 505 // Offset value is corrected accordingly when offset is encoded/decoded. 506 // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics. 507 field bits<21> Offset; 508 let Offset{6-0} = !if(ps.has_offset, offset{6-0}, 509 !if(ps.has_soffset, soffset{6-0}, ?)); 510 let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?); 511 let Inst{52-32} = Offset; 512 513 // soffset 514 let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset), 515 soffset{6-0}, ?); 516} 517 518class SMEM_Real_Load_vi<bits<8> op, string ps> 519 : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps)>; 520 521// The alternative GFX9 SGPR encoding using soffset to encode the 522// offset register. Not available in assembler and goes to the GFX9 523// encoding family to avoid conflicts with the primary SGPR variant. 524class SMEM_Real_SGPR_alt_gfx9 { 525 bit IsGFX9SpecificEncoding = true; 526 bit SOffsetEn = 1; 527 bit Offset = ?; 528 int Subtarget = SIEncodingFamily.GFX9; 529 string AsmVariantName = "NonParsable"; 530} 531 532multiclass SM_Real_Loads_vi<bits<8> op> { 533 defvar ps = NAME; 534 def _IMM_vi : SMEM_Real_Load_vi <op, ps#"_IMM">; 535 def _SGPR_vi : SMEM_Real_Load_vi <op, ps#"_SGPR">; 536 def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR">, 537 SMEM_Real_SGPR_alt_gfx9; 538 let IsGFX9SpecificEncoding = true in 539 def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <op, ps#"_SGPR_IMM">; 540} 541 542class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> { 543 // encoding 544 bits<7> sdata; 545 546 let sdst = ?; 547 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 548} 549 550class SMEM_Real_Store_vi <bits<8> op, string ps> 551 : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)>; 552 553multiclass SM_Real_Stores_vi<bits<8> op> { 554 defvar ps = NAME; 555 def _IMM_vi : SMEM_Real_Store_vi <op, ps#"_IMM">; 556 def _SGPR_vi : SMEM_Real_Store_vi <op, ps#"_SGPR">; 557 def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR">, 558 SMEM_Real_SGPR_alt_gfx9; 559 let IsGFX9SpecificEncoding = true in 560 def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR_IMM">; 561} 562 563multiclass SM_Real_Probe_vi<bits<8> op> { 564 defvar ps = NAME; 565 def _IMM_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 566 def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 567 def _SGPR_alt_gfx9 568 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>, 569 SMEM_Real_SGPR_alt_gfx9; 570 let IsGFX9SpecificEncoding = true in 571 def _SGPR_IMM_gfx9 572 : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 573} 574 575defm S_LOAD_DWORD : SM_Real_Loads_vi <0x00>; 576defm S_LOAD_DWORDX2 : SM_Real_Loads_vi <0x01>; 577defm S_LOAD_DWORDX4 : SM_Real_Loads_vi <0x02>; 578defm S_LOAD_DWORDX8 : SM_Real_Loads_vi <0x03>; 579defm S_LOAD_DWORDX16 : SM_Real_Loads_vi <0x04>; 580defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_vi <0x08>; 581defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_vi <0x09>; 582defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_vi <0x0a>; 583defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_vi <0x0b>; 584defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c>; 585 586defm S_STORE_DWORD : SM_Real_Stores_vi <0x10>; 587defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11>; 588defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12>; 589 590defm S_BUFFER_STORE_DWORD : SM_Real_Stores_vi <0x18>; 591defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_vi <0x19>; 592defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_vi <0x1a>; 593 594// These instructions use same encoding 595def S_DCACHE_INV_vi : SMEM_Real_vi <0x20, S_DCACHE_INV>; 596def S_DCACHE_WB_vi : SMEM_Real_vi <0x21, S_DCACHE_WB>; 597def S_DCACHE_INV_VOL_vi : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>; 598def S_DCACHE_WB_VOL_vi : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>; 599def S_MEMTIME_vi : SMEM_Real_vi <0x24, S_MEMTIME>; 600def S_MEMREALTIME_vi : SMEM_Real_vi <0x25, S_MEMREALTIME>; 601 602defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_vi <0x05>; 603defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_vi <0x06>; 604defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_vi <0x07>; 605 606defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_vi <0x15>; 607defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16>; 608defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17>; 609 610defm S_ATC_PROBE : SM_Real_Probe_vi <0x26>; 611defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27>; 612 613//===----------------------------------------------------------------------===// 614// GFX9 615//===----------------------------------------------------------------------===// 616 617class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps> 618 : SMEM_Real_vi <op, ps>, 619 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 620 621 bits<7> sdata; 622 623 let Constraints = ps.Constraints; 624 let DisableEncoding = ps.DisableEncoding; 625 626 let cpol{CPolBit.GLC} = ps.glc; 627 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 628} 629 630multiclass SM_Real_Atomics_vi<bits<8> op> { 631 defvar ps = NAME; 632 def _IMM_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 633 def _SGPR_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 634 def _SGPR_alt_gfx9 635 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>, 636 SMEM_Real_SGPR_alt_gfx9; 637 let IsGFX9SpecificEncoding = true in 638 def _SGPR_IMM_gfx9 639 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 640 def _IMM_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 641 def _SGPR_RTN_vi : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 642 def _SGPR_RTN_alt_gfx9 643 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>, 644 SMEM_Real_SGPR_alt_gfx9; 645 let IsGFX9SpecificEncoding = true in 646 def _SGPR_IMM_RTN_gfx9 647 : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 648} 649 650defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_vi <0x40>; 651defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x41>; 652defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_vi <0x42>; 653defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_vi <0x43>; 654defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_vi <0x44>; 655defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_vi <0x45>; 656defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_vi <0x46>; 657defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_vi <0x47>; 658defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_vi <0x48>; 659defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_vi <0x49>; 660defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_vi <0x4a>; 661defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_vi <0x4b>; 662defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_vi <0x4c>; 663 664defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0x60>; 665defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0x61>; 666defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0x62>; 667defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0x63>; 668defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0x64>; 669defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0x65>; 670defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0x66>; 671defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0x67>; 672defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0x68>; 673defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0x69>; 674defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0x6a>; 675defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0x6b>; 676defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0x6c>; 677 678defm S_ATOMIC_SWAP : SM_Real_Atomics_vi <0x80>; 679defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_vi <0x81>; 680defm S_ATOMIC_ADD : SM_Real_Atomics_vi <0x82>; 681defm S_ATOMIC_SUB : SM_Real_Atomics_vi <0x83>; 682defm S_ATOMIC_SMIN : SM_Real_Atomics_vi <0x84>; 683defm S_ATOMIC_UMIN : SM_Real_Atomics_vi <0x85>; 684defm S_ATOMIC_SMAX : SM_Real_Atomics_vi <0x86>; 685defm S_ATOMIC_UMAX : SM_Real_Atomics_vi <0x87>; 686defm S_ATOMIC_AND : SM_Real_Atomics_vi <0x88>; 687defm S_ATOMIC_OR : SM_Real_Atomics_vi <0x89>; 688defm S_ATOMIC_XOR : SM_Real_Atomics_vi <0x8a>; 689defm S_ATOMIC_INC : SM_Real_Atomics_vi <0x8b>; 690defm S_ATOMIC_DEC : SM_Real_Atomics_vi <0x8c>; 691 692defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_vi <0xa0>; 693defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_vi <0xa1>; 694defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_vi <0xa2>; 695defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_vi <0xa3>; 696defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_vi <0xa4>; 697defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_vi <0xa5>; 698defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_vi <0xa6>; 699defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_vi <0xa7>; 700defm S_ATOMIC_AND_X2 : SM_Real_Atomics_vi <0xa8>; 701defm S_ATOMIC_OR_X2 : SM_Real_Atomics_vi <0xa9>; 702defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_vi <0xaa>; 703defm S_ATOMIC_INC_X2 : SM_Real_Atomics_vi <0xab>; 704defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_vi <0xac>; 705 706multiclass SM_Real_Discard_vi<bits<8> op> { 707 defvar ps = NAME; 708 def _IMM_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>; 709 def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>; 710 def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>, 711 SMEM_Real_SGPR_alt_gfx9; 712 let IsGFX9SpecificEncoding = true in 713 def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>; 714} 715 716defm S_DCACHE_DISCARD : SM_Real_Discard_vi <0x28>; 717defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29>; 718 719//===----------------------------------------------------------------------===// 720// CI 721//===----------------------------------------------------------------------===// 722 723def smrd_literal_offset : ImmOperand<i32, "SMRDLiteralOffset">; 724 725class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : 726 SM_Real<ps>, 727 Enc64 { 728 729 let AssemblerPredicate = isGFX7Only; 730 let DecoderNamespace = "GFX7"; 731 let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol); 732 733 let Inst{7-0} = 0xff; 734 let Inst{8} = 0; 735 let Inst{14-9} = sbase{6-1}; 736 let Inst{21-15} = sdst{6-0}; 737 let Inst{26-22} = op; 738 let Inst{31-27} = 0x18; //encoding 739 let Inst{63-32} = offset{31-0}; 740} 741 742def S_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>; 743def S_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>; 744def S_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>; 745def S_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>; 746def S_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>; 747def S_BUFFER_LOAD_DWORD_IMM_ci : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>; 748def S_BUFFER_LOAD_DWORDX2_IMM_ci : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>; 749def S_BUFFER_LOAD_DWORDX4_IMM_ci : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>; 750def S_BUFFER_LOAD_DWORDX8_IMM_ci : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>; 751def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>; 752 753class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> 754 : SM_Real<ps> 755 , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> 756 , Enc32 { 757 758 let AssemblerPredicate = isGFX7Only; 759 let DecoderNamespace = "GFX7"; 760 761 let Inst{7-0} = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?)); 762 let Inst{8} = ps.has_offset; 763 let Inst{14-9} = !if(ps.has_sbase, sbase{6-1}, ?); 764 let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?); 765 let Inst{26-22} = op; 766 let Inst{31-27} = 0x18; //encoding 767} 768 769def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>; 770 771//===----------------------------------------------------------------------===// 772// Scalar Memory Patterns 773//===----------------------------------------------------------------------===// 774 775def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> { 776 let GISelPredicateCode = [{ 777 if (!MI.hasOneMemOperand()) 778 return false; 779 if (!isInstrUniform(MI)) 780 return false; 781 782 // FIXME: We should probably be caching this. 783 SmallVector<GEPInfo, 4> AddrInfo; 784 getAddrModeInfo(MI, MRI, AddrInfo); 785 786 if (hasVgprParts(AddrInfo)) 787 return false; 788 return true; 789 }]; 790} 791 792def SMRDImm : ComplexPattern<iPTR, 2, "SelectSMRDImm">; 793def SMRDImm32 : ComplexPattern<iPTR, 2, "SelectSMRDImm32">; 794def SMRDSgpr : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">; 795def SMRDSgprImm : ComplexPattern<iPTR, 3, "SelectSMRDSgprImm">; 796def SMRDBufferImm : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">; 797def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">; 798def SMRDBufferSgprImm : ComplexPattern<iPTR, 2, "SelectSMRDBufferSgprImm">; 799 800multiclass SMRD_Pattern <string Instr, ValueType vt> { 801 802 // 1. IMM offset 803 def : GCNPat < 804 (smrd_load (SMRDImm i64:$sbase, i32:$offset)), 805 (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0)) 806 >; 807 808 // 2. 32-bit IMM offset on CI 809 def : GCNPat < 810 (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)), 811 (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> { 812 let OtherPredicates = [isGFX7Only]; 813 } 814 815 // 3. SGPR offset 816 def : GCNPat < 817 (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)), 818 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0))> { 819 let OtherPredicates = [isNotGFX9Plus]; 820 } 821 def : GCNPat < 822 (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)), 823 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, 0, 0))> { 824 let OtherPredicates = [isGFX9Plus]; 825 } 826 827 // 4. SGPR+IMM offset 828 def : GCNPat < 829 (smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)), 830 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> { 831 let OtherPredicates = [isGFX9Plus]; 832 } 833 834 // 5. No offset 835 def : GCNPat < 836 (vt (smrd_load (i64 SReg_64:$sbase))), 837 (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0)) 838 >; 839} 840 841multiclass SMLoad_Pattern <string Instr, ValueType vt> { 842 // 1. Offset as an immediate 843 def : GCNPat < 844 (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy), 845 (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> { 846 let AddedComplexity = 2; 847 } 848 849 // 2. 32-bit IMM offset on CI 850 def : GCNPat < 851 (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)), 852 (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset, 853 (extract_cpol $cachepolicy))> { 854 let OtherPredicates = [isGFX7Only]; 855 let AddedComplexity = 1; 856 } 857 858 // 3. Offset loaded in an 32bit SGPR 859 def : GCNPat < 860 (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy), 861 (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$soffset, (extract_cpol $cachepolicy)))> { 862 let OtherPredicates = [isNotGFX9Plus]; 863 } 864 def : GCNPat < 865 (SIsbuffer_load v4i32:$sbase, i32:$soffset, timm:$cachepolicy), 866 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, 0, (extract_cpol $cachepolicy)))> { 867 let OtherPredicates = [isGFX9Plus]; 868 } 869 870 // 4. Offset as an 32-bit SGPR + immediate 871 def : GCNPat < 872 (SIsbuffer_load v4i32:$sbase, (SMRDBufferSgprImm i32:$soffset, i32:$offset), 873 timm:$cachepolicy), 874 (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") SReg_128:$sbase, SReg_32:$soffset, i32imm:$offset, 875 (extract_cpol $cachepolicy)))> { 876 let OtherPredicates = [isGFX9Plus]; 877 } 878} 879 880// Global and constant loads can be selected to either MUBUF or SMRD 881// instructions, but SMRD instructions are faster so we want the instruction 882// selector to prefer those. 883let AddedComplexity = 100 in { 884 885foreach vt = Reg32Types.types in { 886defm : SMRD_Pattern <"S_LOAD_DWORD", vt>; 887} 888 889foreach vt = SReg_64.RegTypes in { 890defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>; 891} 892 893foreach vt = SReg_128.RegTypes in { 894defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>; 895} 896 897foreach vt = SReg_256.RegTypes in { 898defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>; 899} 900 901foreach vt = SReg_512.RegTypes in { 902defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>; 903} 904 905} // End let AddedComplexity = 100 906 907defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", i32>; 908defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2i32>; 909defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4i32>; 910defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8i32>; 911defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16i32>; 912 913defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD", f32>; 914defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2", v2f32>; 915defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4", v4f32>; 916defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8", v8f32>; 917defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16", v16f32>; 918 919let OtherPredicates = [HasSMemTimeInst] in { 920def : GCNPat < 921 (i64 (readcyclecounter)), 922 (S_MEMTIME) 923>; 924} // let OtherPredicates = [HasSMemTimeInst] 925 926let OtherPredicates = [HasShaderCyclesRegister] in { 927def : GCNPat < 928 (i64 (readcyclecounter)), 929 (REG_SEQUENCE SReg_64, 930 (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0, 931 (S_MOV_B32 (i32 0)), sub1)> { 932 // Prefer this to s_memtime because it has lower and more predictable latency. 933 let AddedComplexity = 1; 934} 935} // let OtherPredicates = [HasShaderCyclesRegister] 936 937//===----------------------------------------------------------------------===// 938// GFX10. 939//===----------------------------------------------------------------------===// 940 941class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName, 942 int subtarget, RegisterWithSubRegs sgpr_null> : 943 SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 { 944 let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); 945 let Inst{12-6} = !if(ps.has_sdst, sdst{6-0}, ?); 946 let Inst{25-18} = op; 947 let Inst{31-26} = 0x3d; 948 // There are SMEM instructions that do not employ any of the offset 949 // fields, in which case we need them to remain undefined. 950 let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?)); 951 let Inst{63-57} = !if(ps.has_soffset, soffset{6-0}, 952 !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?)); 953} 954 955class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> 956 : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10, 957 SGPR_NULL_gfxpre11> { 958 let AssemblerPredicate = isGFX10Only; 959 let DecoderNamespace = "GFX10"; 960 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?); 961 let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?); 962} 963 964class SMEM_Real_Load_gfx10<bits<8> op, string ps> 965 : SMEM_Real_gfx10<op, !cast<SM_Pseudo>(ps)>; 966 967multiclass SM_Real_Loads_gfx10<bits<8> op> { 968 defvar ps = NAME; 969 def _IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_IMM">; 970 def _SGPR_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR">; 971 def _SGPR_IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps#"_SGPR_IMM">; 972} 973 974class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> { 975 bits<7> sdata; 976 977 let sdst = ?; 978 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 979} 980 981multiclass SM_Real_Stores_gfx10<bits<8> op> { 982 defvar ps = NAME; 983 defvar immPs = !cast<SM_Store_Pseudo>(ps#_IMM); 984 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs>; 985 986 defvar sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR); 987 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs>; 988 989 defvar sgprImmPs = !cast<SM_Store_Pseudo>(ps#_SGPR_IMM); 990 def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, sgprImmPs>; 991} 992 993defm S_LOAD_DWORD : SM_Real_Loads_gfx10<0x000>; 994defm S_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x001>; 995defm S_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x002>; 996defm S_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x003>; 997defm S_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x004>; 998 999let SubtargetPredicate = HasScalarFlatScratchInsts in { 1000defm S_SCRATCH_LOAD_DWORD : SM_Real_Loads_gfx10<0x005>; 1001defm S_SCRATCH_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x006>; 1002defm S_SCRATCH_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x007>; 1003} // End SubtargetPredicate = HasScalarFlatScratchInsts 1004 1005defm S_BUFFER_LOAD_DWORD : SM_Real_Loads_gfx10<0x008>; 1006defm S_BUFFER_LOAD_DWORDX2 : SM_Real_Loads_gfx10<0x009>; 1007defm S_BUFFER_LOAD_DWORDX4 : SM_Real_Loads_gfx10<0x00a>; 1008defm S_BUFFER_LOAD_DWORDX8 : SM_Real_Loads_gfx10<0x00b>; 1009defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_gfx10<0x00c>; 1010 1011let SubtargetPredicate = HasScalarStores in { 1012defm S_STORE_DWORD : SM_Real_Stores_gfx10<0x010>; 1013defm S_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x011>; 1014defm S_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x012>; 1015let OtherPredicates = [HasScalarFlatScratchInsts] in { 1016defm S_SCRATCH_STORE_DWORD : SM_Real_Stores_gfx10<0x015>; 1017defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016>; 1018defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017>; 1019} // End OtherPredicates = [HasScalarFlatScratchInsts] 1020defm S_BUFFER_STORE_DWORD : SM_Real_Stores_gfx10<0x018>; 1021defm S_BUFFER_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x019>; 1022defm S_BUFFER_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x01a>; 1023} // End SubtargetPredicate = HasScalarStores 1024 1025def S_MEMREALTIME_gfx10 : SMEM_Real_gfx10<0x025, S_MEMREALTIME>; 1026def S_MEMTIME_gfx10 : SMEM_Real_gfx10<0x024, S_MEMTIME>; 1027def S_GL1_INV_gfx10 : SMEM_Real_gfx10<0x01f, S_GL1_INV>; 1028def S_GET_WAVEID_IN_WORKGROUP_gfx10 : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>; 1029def S_DCACHE_INV_gfx10 : SMEM_Real_gfx10<0x020, S_DCACHE_INV>; 1030 1031let SubtargetPredicate = HasScalarStores in { 1032def S_DCACHE_WB_gfx10 : SMEM_Real_gfx10<0x021, S_DCACHE_WB>; 1033} // End SubtargetPredicate = HasScalarStores 1034 1035multiclass SM_Real_Probe_gfx10<bits<8> op> { 1036 defvar ps = NAME; 1037 def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1038 def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1039 def _SGPR_IMM_gfx10 1040 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1041} 1042 1043defm S_ATC_PROBE : SM_Real_Probe_gfx10 <0x26>; 1044defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27>; 1045 1046class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps> 1047 : SMEM_Real_gfx10 <op, ps>, 1048 AtomicNoRet <!subst("_RTN","",NAME), ps.glc> { 1049 1050 bits<7> sdata; 1051 1052 let Constraints = ps.Constraints; 1053 let DisableEncoding = ps.DisableEncoding; 1054 1055 let cpol{CPolBit.GLC} = ps.glc; 1056 1057 let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1058 let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0}); 1059} 1060 1061multiclass SM_Real_Atomics_gfx10<bits<8> op> { 1062 defvar ps = NAME; 1063 def _IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>; 1064 def _SGPR_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>; 1065 def _SGPR_IMM_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>; 1066 def _IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>; 1067 def _SGPR_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>; 1068 def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>; 1069} 1070 1071let SubtargetPredicate = HasScalarAtomics in { 1072 1073defm S_BUFFER_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x40>; 1074defm S_BUFFER_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x41>; 1075defm S_BUFFER_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x42>; 1076defm S_BUFFER_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x43>; 1077defm S_BUFFER_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x44>; 1078defm S_BUFFER_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x45>; 1079defm S_BUFFER_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x46>; 1080defm S_BUFFER_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x47>; 1081defm S_BUFFER_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x48>; 1082defm S_BUFFER_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x49>; 1083defm S_BUFFER_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x4a>; 1084defm S_BUFFER_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x4b>; 1085defm S_BUFFER_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x4c>; 1086 1087defm S_BUFFER_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0x60>; 1088defm S_BUFFER_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0x61>; 1089defm S_BUFFER_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0x62>; 1090defm S_BUFFER_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0x63>; 1091defm S_BUFFER_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0x64>; 1092defm S_BUFFER_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0x65>; 1093defm S_BUFFER_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0x66>; 1094defm S_BUFFER_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0x67>; 1095defm S_BUFFER_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0x68>; 1096defm S_BUFFER_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0x69>; 1097defm S_BUFFER_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0x6a>; 1098defm S_BUFFER_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0x6b>; 1099defm S_BUFFER_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0x6c>; 1100 1101defm S_ATOMIC_SWAP : SM_Real_Atomics_gfx10 <0x80>; 1102defm S_ATOMIC_CMPSWAP : SM_Real_Atomics_gfx10 <0x81>; 1103defm S_ATOMIC_ADD : SM_Real_Atomics_gfx10 <0x82>; 1104defm S_ATOMIC_SUB : SM_Real_Atomics_gfx10 <0x83>; 1105defm S_ATOMIC_SMIN : SM_Real_Atomics_gfx10 <0x84>; 1106defm S_ATOMIC_UMIN : SM_Real_Atomics_gfx10 <0x85>; 1107defm S_ATOMIC_SMAX : SM_Real_Atomics_gfx10 <0x86>; 1108defm S_ATOMIC_UMAX : SM_Real_Atomics_gfx10 <0x87>; 1109defm S_ATOMIC_AND : SM_Real_Atomics_gfx10 <0x88>; 1110defm S_ATOMIC_OR : SM_Real_Atomics_gfx10 <0x89>; 1111defm S_ATOMIC_XOR : SM_Real_Atomics_gfx10 <0x8a>; 1112defm S_ATOMIC_INC : SM_Real_Atomics_gfx10 <0x8b>; 1113defm S_ATOMIC_DEC : SM_Real_Atomics_gfx10 <0x8c>; 1114 1115defm S_ATOMIC_SWAP_X2 : SM_Real_Atomics_gfx10 <0xa0>; 1116defm S_ATOMIC_CMPSWAP_X2 : SM_Real_Atomics_gfx10 <0xa1>; 1117defm S_ATOMIC_ADD_X2 : SM_Real_Atomics_gfx10 <0xa2>; 1118defm S_ATOMIC_SUB_X2 : SM_Real_Atomics_gfx10 <0xa3>; 1119defm S_ATOMIC_SMIN_X2 : SM_Real_Atomics_gfx10 <0xa4>; 1120defm S_ATOMIC_UMIN_X2 : SM_Real_Atomics_gfx10 <0xa5>; 1121defm S_ATOMIC_SMAX_X2 : SM_Real_Atomics_gfx10 <0xa6>; 1122defm S_ATOMIC_UMAX_X2 : SM_Real_Atomics_gfx10 <0xa7>; 1123defm S_ATOMIC_AND_X2 : SM_Real_Atomics_gfx10 <0xa8>; 1124defm S_ATOMIC_OR_X2 : SM_Real_Atomics_gfx10 <0xa9>; 1125defm S_ATOMIC_XOR_X2 : SM_Real_Atomics_gfx10 <0xaa>; 1126defm S_ATOMIC_INC_X2 : SM_Real_Atomics_gfx10 <0xab>; 1127defm S_ATOMIC_DEC_X2 : SM_Real_Atomics_gfx10 <0xac>; 1128 1129multiclass SM_Real_Discard_gfx10<bits<8> op> { 1130 defvar ps = NAME; 1131 def _IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>; 1132 def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>; 1133 def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>; 1134} 1135 1136defm S_DCACHE_DISCARD : SM_Real_Discard_gfx10 <0x28>; 1137defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29>; 1138 1139} // End SubtargetPredicate = HasScalarAtomics 1140 1141def SMInfoTable : GenericTable { 1142 let FilterClass = "SM_Real"; 1143 let CppTypeName = "SMInfo"; 1144 let Fields = ["Opcode", "is_buffer"]; 1145 1146 let PrimaryKey = ["Opcode"]; 1147 let PrimaryKeyName = "getSMEMOpcodeHelper"; 1148} 1149 1150//===----------------------------------------------------------------------===// 1151// GFX11. 1152//===----------------------------------------------------------------------===// 1153 1154class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> : 1155 SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11, 1156 SGPR_NULL_gfx11plus> { 1157 let AssemblerPredicate = isGFX11Plus; 1158 let DecoderNamespace = "GFX11"; 1159 let Inst{13} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0); 1160 let Inst{14} = !if(ps.has_glc, cpol{CPolBit.GLC}, 0); 1161} 1162 1163class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName> : 1164 SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps), opName>; 1165 1166multiclass SM_Real_Loads_gfx11<bits<8> op, string ps> { 1167 defvar opName = !tolower(NAME); 1168 def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_IMM", opName>; 1169 def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR", opName>; 1170 def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps#"_SGPR_IMM", opName>; 1171 def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>, 1172 Requires<[isGFX11Plus]>; 1173} 1174 1175defm S_LOAD_B32 : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD">; 1176defm S_LOAD_B64 : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2">; 1177defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4">; 1178defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8">; 1179defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16">; 1180 1181defm S_BUFFER_LOAD_B32 : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD">; 1182defm S_BUFFER_LOAD_B64 : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2">; 1183defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4">; 1184defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8">; 1185defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16">; 1186 1187def S_GL1_INV_gfx11 : SMEM_Real_gfx11<0x020, S_GL1_INV>; 1188def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>; 1189 1190class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> { 1191 // encoding 1192 bits<7> sdata; 1193 1194 let sdst = ?; 1195 let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?); 1196} 1197 1198multiclass SM_Real_Probe_gfx11<bits<8> op> { 1199 defvar ps = NAME; 1200 def _IMM_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>; 1201 def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>; 1202 def _SGPR_IMM_gfx11 1203 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>; 1204} 1205 1206defm S_ATC_PROBE : SM_Real_Probe_gfx11 <0x22>; 1207defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23>; 1208