xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/SMInstructions.td (revision e6bfd18d21b225af6a0ed67ceeaf1293b7b9eba5)
1//===---- SMInstructions.td - Scalar Memory Instruction Definitions -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9def smrd_offset_8 : NamedOperandU32<"SMRDOffset8",
10                                  NamedMatchClass<"SMRDOffset8">> {
11  let OperandType = "OPERAND_IMMEDIATE";
12}
13
14class SMEMOffset : NamedOperandU32<"SMEMOffset",
15                                   NamedMatchClass<"SMEMOffset">> {
16  let OperandType = "OPERAND_IMMEDIATE";
17  let EncoderMethod = "getSMEMOffsetEncoding";
18  let DecoderMethod = "decodeSMEMOffset";
19}
20
21def smem_offset : SMEMOffset;
22
23def smem_offset_mod : SMEMOffset {
24  let PrintMethod = "printSMEMOffsetMod";
25}
26
27//===----------------------------------------------------------------------===//
28// Scalar Memory classes
29//===----------------------------------------------------------------------===//
30
31class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
32  InstSI <outs, ins, "", pattern>,
33  SIMCInstr<opName, SIEncodingFamily.NONE> {
34  let isPseudo = 1;
35  let isCodeGenOnly = 1;
36
37  let LGKM_CNT = 1;
38  let SMRD = 1;
39  let mayStore = 0;
40  let mayLoad = 1;
41  let hasSideEffects = 0;
42  let UseNamedOperandTable = 1;
43  let SchedRW = [WriteSMEM];
44
45  string Mnemonic = opName;
46  string AsmOperands = asmOps;
47
48  bits<1> has_sbase = 1;
49  bits<1> has_sdst = 1;
50  bit has_glc = 0;
51  bit has_dlc = 0;
52  bit has_offset = 0;
53  bit has_soffset = 0;
54  bit is_buffer = 0;
55}
56
57class SM_Real <SM_Pseudo ps, string opName = ps.Mnemonic>
58  : InstSI<ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands> {
59
60  let isPseudo = 0;
61  let isCodeGenOnly = 0;
62
63  Instruction Opcode = !cast<Instruction>(NAME);
64
65  // copy relevant pseudo op flags
66  let LGKM_CNT             = ps.LGKM_CNT;
67  let SMRD                 = ps.SMRD;
68  let mayStore             = ps.mayStore;
69  let mayLoad              = ps.mayLoad;
70  let hasSideEffects       = ps.hasSideEffects;
71  let UseNamedOperandTable = ps.UseNamedOperandTable;
72  let SchedRW              = ps.SchedRW;
73  let SubtargetPredicate   = ps.SubtargetPredicate;
74  let AsmMatchConverter    = ps.AsmMatchConverter;
75  let IsAtomicRet          = ps.IsAtomicRet;
76  let IsAtomicNoRet        = ps.IsAtomicNoRet;
77
78  let TSFlags = ps.TSFlags;
79
80  bit is_buffer = ps.is_buffer;
81
82  // encoding
83  bits<7>  sbase;
84  bits<7>  sdst;
85  bits<32> offset;
86  bits<8>  soffset;
87  bits<5> cpol;
88}
89
90class OffsetMode<bit hasOffset, bit hasSOffset, string variant,
91                 dag ins, string asm> {
92  bit HasOffset = hasOffset;
93  bit HasSOffset = hasSOffset;
94  string Variant = variant;
95  dag Ins = ins;
96  string Asm = asm;
97}
98
99def IMM_Offset : OffsetMode<1, 0, "_IMM", (ins smem_offset:$offset), "$offset">;
100def SGPR_Offset : OffsetMode<0, 1, "_SGPR", (ins SReg_32:$soffset), "$soffset">;
101def SGPR_IMM_Offset : OffsetMode<1, 1, "_SGPR_IMM",
102                                 (ins SReg_32:$soffset, smem_offset_mod:$offset),
103                                 "$soffset$offset">;
104
105class SM_Probe_Pseudo <string opName, string variant, RegisterClass baseClass,
106                       dag offsets, string asmOffsets,
107                       bit hasOffset, bit hasSOffset>
108  : SM_Pseudo<opName, (outs),
109              !con((ins i8imm:$sdata, baseClass:$sbase), offsets),
110              " $sdata, $sbase, " # asmOffsets> {
111  let mayLoad = 0;
112  let mayStore = 0;
113  let has_glc = 0;
114  let LGKM_CNT = 0;
115  let ScalarStore = 0;
116  let hasSideEffects = 1;
117  let has_offset = hasOffset;
118  let has_soffset = hasSOffset;
119  let PseudoInstr = opName # variant;
120}
121
122class SM_Load_Pseudo <string opName, RegisterClass baseClass,
123                      RegisterClass dstClass, OffsetMode offsets>
124  : SM_Pseudo<opName, (outs dstClass:$sdst),
125              !con((ins baseClass:$sbase), offsets.Ins, (ins CPol:$cpol)),
126              " $sdst, $sbase, " # offsets.Asm # "$cpol", []> {
127  RegisterClass BaseClass = baseClass;
128  let mayLoad = 1;
129  let mayStore = 0;
130  let has_glc = 1;
131  let has_dlc = 1;
132  let has_offset = offsets.HasOffset;
133  let has_soffset = offsets.HasSOffset;
134  let PseudoInstr = opName # offsets.Variant;
135}
136
137class SM_Store_Pseudo <string opName, RegisterClass baseClass,
138                       RegisterClass srcClass, dag ins, string asmOps>
139  : SM_Pseudo<opName, (outs), ins, asmOps, []> {
140  RegisterClass BaseClass = baseClass;
141  RegisterClass SrcClass = srcClass;
142  let mayLoad = 0;
143  let mayStore = 1;
144  let has_glc = 1;
145  let has_dlc = 1;
146  let ScalarStore = 1;
147}
148
149class SM_Discard_Pseudo <string opName, string variant, dag offsets,
150                         string asmOffsets, bit hasOffset, bit hasSOffset>
151  : SM_Pseudo<opName, (outs), !con((ins SReg_64:$sbase), offsets),
152              " $sbase, " # asmOffsets> {
153  let mayLoad = 0;
154  let mayStore = 0;
155  let has_glc = 0;
156  let has_sdst = 0;
157  let ScalarStore = 0;
158  let hasSideEffects = 1;
159  let has_offset = hasOffset;
160  let has_soffset = hasSOffset;
161  let PseudoInstr = opName # variant;
162}
163
164multiclass SM_Pseudo_Loads<string opName,
165                           RegisterClass baseClass,
166                           RegisterClass dstClass> {
167  def _IMM : SM_Load_Pseudo <opName, baseClass, dstClass, IMM_Offset>;
168  def _SGPR : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_Offset>;
169  def _SGPR_IMM : SM_Load_Pseudo <opName, baseClass, dstClass, SGPR_IMM_Offset>;
170}
171
172multiclass SM_Pseudo_Stores<string opName,
173                           RegisterClass baseClass,
174                           RegisterClass srcClass> {
175  def _IMM : SM_Store_Pseudo <opName, baseClass, srcClass,
176    (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, CPol:$cpol),
177    " $sdata, $sbase, $offset$cpol"> {
178    let has_offset = 1;
179    let PseudoInstr = opName # "_IMM";
180  }
181
182  def _SGPR : SM_Store_Pseudo <opName, baseClass, srcClass,
183    (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, CPol:$cpol),
184    " $sdata, $sbase, $soffset$cpol"> {
185    let has_soffset = 1;
186    let PseudoInstr = opName # "_SGPR";
187  }
188
189  def _SGPR_IMM : SM_Store_Pseudo <opName, baseClass, srcClass,
190    (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soffset, i32imm:$offset,
191     CPol:$cpol),
192    " $sdata, $sbase, $soffset$offset$cpol"> {
193    let has_offset = 1;
194    let has_soffset = 1;
195    let PseudoInstr = opName # "_SGPR_IMM";
196  }
197}
198
199multiclass SM_Pseudo_Discards<string opName> {
200  def _IMM  : SM_Discard_Pseudo <opName, "_IMM",
201    (ins smem_offset:$offset), "$offset", 1, 0>;
202  def _SGPR : SM_Discard_Pseudo <opName, "_SGPR",
203    (ins SReg_32:$soffset), "$soffset", 0, 1>;
204  def _SGPR_IMM : SM_Discard_Pseudo <opName, "_SGPR_IMM",
205    (ins SReg_32:$soffset, smem_offset_mod:$offset), "$soffset$offset", 1, 1>;
206}
207
208class SM_Time_Pseudo<string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
209  opName, (outs SReg_64_XEXEC:$sdst), (ins),
210  " $sdst", [(set i64:$sdst, (node))]> {
211  let hasSideEffects = 1;
212
213  let mayStore = 0;
214  let mayLoad = 0;
215  let has_sbase = 0;
216}
217
218class SM_Inval_Pseudo <string opName, SDPatternOperator node = null_frag> : SM_Pseudo<
219  opName, (outs), (ins), "", [(node)]> {
220  let hasSideEffects = 1;
221  let mayLoad = 0;
222  let mayStore = 0;
223  let has_sdst = 0;
224  let has_sbase = 0;
225}
226
227multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> {
228  def _IMM  : SM_Probe_Pseudo <opName, "_IMM", baseClass,
229    (ins smem_offset:$offset), "$offset", 1, 0>;
230  def _SGPR : SM_Probe_Pseudo <opName, "_SGPR", baseClass,
231    (ins SReg_32:$soffset), "$soffset", 0, 1>;
232  def _SGPR_IMM : SM_Probe_Pseudo <opName, "_SGPR_IMM", baseClass,
233    (ins SReg_32:$soffset, smem_offset_mod:$offset), "$soffset$offset", 1, 1>;
234}
235
236class SM_WaveId_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
237  opName, (outs SReg_32_XM0_XEXEC:$sdst), (ins),
238  " $sdst", [(set i32:$sdst, (node))]> {
239  let hasSideEffects = 1;
240  let mayStore = 0;
241  let mayLoad = 0;
242  let has_sbase = 0;
243}
244
245//===----------------------------------------------------------------------===//
246// Scalar Atomic Memory Classes
247//===----------------------------------------------------------------------===//
248
249class SM_Atomic_Pseudo <string opName,
250                        dag outs, dag ins, string asmOps, bit isRet>
251  : SM_Pseudo<opName, outs, ins, asmOps, []> {
252
253  bit glc = isRet;
254
255  let mayLoad = 1;
256  let mayStore = 1;
257  let has_glc = 1;
258  let has_dlc = 1;
259  let has_soffset = 1;
260
261  // Should these be set?
262  let ScalarStore = 1;
263  let hasSideEffects = 1;
264  let maybeAtomic = 1;
265
266  let IsAtomicNoRet = !not(isRet);
267  let IsAtomicRet = isRet;
268
269  let AsmMatchConverter = "cvtSMEMAtomic";
270}
271
272class SM_Pseudo_Atomic<string opName,
273                       RegisterClass baseClass,
274                       RegisterClass dataClass,
275                       OffsetMode offsets,
276                       bit isRet,
277                       string opNameWithSuffix =
278                         opName # offsets.Variant # !if(isRet, "_RTN", ""),
279                       Operand CPolTy = !if(isRet, CPol_GLC1, CPol)> :
280  SM_Atomic_Pseudo<opName,
281                   !if(isRet, (outs dataClass:$sdst), (outs)),
282                   !con((ins dataClass:$sdata, baseClass:$sbase), offsets.Ins,
283                        (ins CPolTy:$cpol)),
284                   !if(isRet, " $sdst", " $sdata") #
285                     ", $sbase, " # offsets.Asm # "$cpol",
286                   isRet>,
287  AtomicNoRet <opNameWithSuffix, isRet> {
288  let has_offset = offsets.HasOffset;
289  let has_soffset = offsets.HasSOffset;
290  let PseudoInstr = opNameWithSuffix;
291
292  let Constraints = !if(isRet, "$sdst = $sdata", "");
293  let DisableEncoding = !if(isRet, "$sdata", "");
294}
295
296multiclass SM_Pseudo_Atomics<string opName,
297                             RegisterClass baseClass,
298                             RegisterClass dataClass> {
299  def _IMM      : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 0>;
300  def _SGPR     : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 0>;
301  def _SGPR_IMM : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 0>;
302  def _IMM_RTN  : SM_Pseudo_Atomic <opName, baseClass, dataClass, IMM_Offset, 1>;
303  def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_Offset, 1>;
304  def _SGPR_IMM_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, SGPR_IMM_Offset, 1>;
305}
306
307//===----------------------------------------------------------------------===//
308// Scalar Memory Instructions
309//===----------------------------------------------------------------------===//
310
311// We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit
312// SMRD instructions, because the SReg_32_XM0 register class does not include M0
313// and writing to M0 from an SMRD instruction will hang the GPU.
314
315// XXX - SMEM instructions do not allow exec for data operand, but
316// does sdst for SMRD on SI/CI?
317defm S_LOAD_DWORD    : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>;
318defm S_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>;
319defm S_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>;
320defm S_LOAD_DWORDX8  : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>;
321defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>;
322
323let is_buffer = 1 in {
324defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <
325  "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC
326>;
327
328// FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on
329// SI/CI, bit disallowed for SMEM on VI.
330defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <
331  "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC
332>;
333
334defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <
335  "s_buffer_load_dwordx4", SReg_128, SReg_128
336>;
337
338defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <
339  "s_buffer_load_dwordx8", SReg_128, SReg_256
340>;
341
342defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <
343  "s_buffer_load_dwordx16", SReg_128, SReg_512
344>;
345}
346
347let SubtargetPredicate = HasScalarStores in {
348defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>;
349defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>;
350defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>;
351
352let is_buffer = 1 in {
353defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <
354  "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC
355>;
356
357defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <
358  "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC
359>;
360
361defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <
362  "s_buffer_store_dwordx4", SReg_128, SReg_128
363>;
364}
365} // End SubtargetPredicate = HasScalarStores
366
367let SubtargetPredicate = HasSMemTimeInst in
368def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;
369def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;
370
371let SubtargetPredicate = isGFX7GFX8GFX9 in {
372def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
373} // let SubtargetPredicate = isGFX7GFX8GFX9
374
375let SubtargetPredicate = isGFX8Plus in {
376let OtherPredicates = [HasScalarStores] in {
377def S_DCACHE_WB     : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>;
378def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;
379} // End OtherPredicates = [HasScalarStores]
380
381defm S_ATC_PROBE        : SM_Pseudo_Probe <"s_atc_probe", SReg_64>;
382let is_buffer = 1 in {
383defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>;
384}
385} // SubtargetPredicate = isGFX8Plus
386
387let SubtargetPredicate = HasSMemRealTime in
388def S_MEMREALTIME   : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>;
389
390let SubtargetPredicate = isGFX10Plus in
391def S_GL1_INV : SM_Inval_Pseudo<"s_gl1_inv">;
392let SubtargetPredicate = HasGetWaveIdInst in
393def S_GET_WAVEID_IN_WORKGROUP : SM_WaveId_Pseudo <"s_get_waveid_in_workgroup", int_amdgcn_s_get_waveid_in_workgroup>;
394
395
396let SubtargetPredicate = HasScalarFlatScratchInsts, Uses = [FLAT_SCR] in {
397defm S_SCRATCH_LOAD_DWORD    : SM_Pseudo_Loads <"s_scratch_load_dword",   SReg_64, SReg_32_XM0_XEXEC>;
398defm S_SCRATCH_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>;
399defm S_SCRATCH_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>;
400
401defm S_SCRATCH_STORE_DWORD   : SM_Pseudo_Stores <"s_scratch_store_dword",   SReg_64, SReg_32_XM0_XEXEC>;
402defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>;
403defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>;
404} // SubtargetPredicate = HasScalarFlatScratchInsts
405
406let SubtargetPredicate = HasScalarAtomics in {
407
408let is_buffer = 1 in {
409defm S_BUFFER_ATOMIC_SWAP         : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>;
410defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>;
411defm S_BUFFER_ATOMIC_ADD          : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>;
412defm S_BUFFER_ATOMIC_SUB          : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>;
413defm S_BUFFER_ATOMIC_SMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>;
414defm S_BUFFER_ATOMIC_UMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>;
415defm S_BUFFER_ATOMIC_SMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>;
416defm S_BUFFER_ATOMIC_UMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>;
417defm S_BUFFER_ATOMIC_AND          : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>;
418defm S_BUFFER_ATOMIC_OR           : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>;
419defm S_BUFFER_ATOMIC_XOR          : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>;
420defm S_BUFFER_ATOMIC_INC          : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>;
421defm S_BUFFER_ATOMIC_DEC          : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>;
422
423defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>;
424defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>;
425defm S_BUFFER_ATOMIC_ADD_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>;
426defm S_BUFFER_ATOMIC_SUB_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>;
427defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>;
428defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>;
429defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>;
430defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>;
431defm S_BUFFER_ATOMIC_AND_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>;
432defm S_BUFFER_ATOMIC_OR_X2        : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>;
433defm S_BUFFER_ATOMIC_XOR_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>;
434defm S_BUFFER_ATOMIC_INC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>;
435defm S_BUFFER_ATOMIC_DEC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>;
436}
437
438defm S_ATOMIC_SWAP                : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>;
439defm S_ATOMIC_CMPSWAP             : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>;
440defm S_ATOMIC_ADD                 : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>;
441defm S_ATOMIC_SUB                 : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>;
442defm S_ATOMIC_SMIN                : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>;
443defm S_ATOMIC_UMIN                : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>;
444defm S_ATOMIC_SMAX                : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>;
445defm S_ATOMIC_UMAX                : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>;
446defm S_ATOMIC_AND                 : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>;
447defm S_ATOMIC_OR                  : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>;
448defm S_ATOMIC_XOR                 : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>;
449defm S_ATOMIC_INC                 : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>;
450defm S_ATOMIC_DEC                 : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>;
451
452defm S_ATOMIC_SWAP_X2             : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>;
453defm S_ATOMIC_CMPSWAP_X2          : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>;
454defm S_ATOMIC_ADD_X2              : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>;
455defm S_ATOMIC_SUB_X2              : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>;
456defm S_ATOMIC_SMIN_X2             : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>;
457defm S_ATOMIC_UMIN_X2             : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>;
458defm S_ATOMIC_SMAX_X2             : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>;
459defm S_ATOMIC_UMAX_X2             : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>;
460defm S_ATOMIC_AND_X2              : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>;
461defm S_ATOMIC_OR_X2               : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>;
462defm S_ATOMIC_XOR_X2              : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>;
463defm S_ATOMIC_INC_X2              : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>;
464defm S_ATOMIC_DEC_X2              : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>;
465
466} // let SubtargetPredicate = HasScalarAtomics
467
468let SubtargetPredicate = HasScalarAtomics in {
469defm S_DCACHE_DISCARD    : SM_Pseudo_Discards <"s_dcache_discard">;
470defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">;
471}
472
473//===----------------------------------------------------------------------===//
474// Targets
475//===----------------------------------------------------------------------===//
476
477//===----------------------------------------------------------------------===//
478// SI
479//===----------------------------------------------------------------------===//
480
481class SMRD_Real_si <bits<5> op, SM_Pseudo ps>
482  : SM_Real<ps>
483  , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>
484  , Enc32 {
485
486  let AssemblerPredicate = isGFX6GFX7;
487  let DecoderNamespace = "GFX6GFX7";
488
489  let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));
490  let Inst{8}     = ps.has_offset;
491  let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
492  let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
493  let Inst{26-22} = op;
494  let Inst{31-27} = 0x18; //encoding
495}
496
497multiclass SM_Real_Loads_si<bits<5> op, string ps,
498                            SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
499                            SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
500
501  def _IMM_si : SMRD_Real_si <op, immPs> {
502    let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, CPol:$cpol);
503  }
504
505  def _SGPR_si : SMRD_Real_si <op, sgprPs> {
506    let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
507  }
508
509}
510
511defm S_LOAD_DWORD           : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">;
512defm S_LOAD_DWORDX2         : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">;
513defm S_LOAD_DWORDX4         : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">;
514defm S_LOAD_DWORDX8         : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">;
515defm S_LOAD_DWORDX16        : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">;
516defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">;
517defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">;
518defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">;
519defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">;
520defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">;
521
522def S_MEMTIME_si    : SMRD_Real_si <0x1e, S_MEMTIME>;
523def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>;
524
525
526//===----------------------------------------------------------------------===//
527// VI and GFX9.
528//===----------------------------------------------------------------------===//
529
530class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>
531  : SM_Real<ps>
532  , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI>
533  , Enc64 {
534  field bit IsGFX9SpecificEncoding = false;
535  let AssemblerPredicate = !if(IsGFX9SpecificEncoding, isGFX9Only, isGFX8GFX9);
536  let DecoderNamespace = "GFX8";
537
538  let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
539  let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
540
541  // Note that for GFX9 instructions with immediate offsets, soffset_en
542  // must be defined, whereas in GFX8 it's undefined in all cases,
543  // meaning GFX9 is not perfectly backward-compatible with GFX8, despite
544  // documentation suggesting otherwise.
545  field bit SOffsetEn = !if(IsGFX9SpecificEncoding,
546    !if(ps.has_offset, ps.has_soffset, !if(ps.has_soffset, 0, ?)),
547    ?);
548  let Inst{14} = SOffsetEn;
549
550  let Inst{16} = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);
551
552  // imm
553  // TODO: Shall not be defined if the instruction has no offset nor
554  // soffset.
555  let Inst{17} = ps.has_offset;
556
557  let Inst{25-18} = op;
558  let Inst{31-26} = 0x30; //encoding
559
560  // VI supports 20-bit unsigned offsets while GFX9+ supports 21-bit signed.
561  // Offset value is corrected accordingly when offset is encoded/decoded.
562  // TODO: Forbid non-M0 register offsets for GFX8 stores and atomics.
563  field bits<21> Offset;
564  let Offset{6-0} = !if(ps.has_offset, offset{6-0},
565                                       !if(ps.has_soffset, soffset{6-0}, ?));
566  let Offset{20-7} = !if(ps.has_offset, offset{20-7}, ?);
567  let Inst{52-32} = Offset;
568
569  // soffset
570  let Inst{63-57} = !if(!and(IsGFX9SpecificEncoding, ps.has_soffset),
571                        soffset{6-0}, ?);
572}
573
574class SMEM_Real_Load_vi<bits<8> op, string ps, OffsetMode offsets>
575    : SMEM_Real_vi<op, !cast<SM_Pseudo>(ps # offsets.Variant)> {
576  RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass;
577  let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol));
578}
579
580// The alternative GFX9 SGPR encoding using soffset to encode the
581// offset register. Not available in assembler and goes to the GFX9
582// encoding family to avoid conflicts with the primary SGPR variant.
583class SMEM_Real_SGPR_alt_gfx9 {
584  bit IsGFX9SpecificEncoding = true;
585  bit SOffsetEn = 1;
586  bit Offset = ?;
587  int Subtarget = SIEncodingFamily.GFX9;
588  string AsmVariantName = "NonParsable";
589}
590
591multiclass SM_Real_Loads_vi<bits<8> op, string ps> {
592  def _IMM_vi : SMEM_Real_Load_vi <op, ps, IMM_Offset>;
593  def _SGPR_vi : SMEM_Real_Load_vi <op, ps, SGPR_Offset>;
594  def _SGPR_alt_gfx9 : SMEM_Real_Load_vi <op, ps, SGPR_Offset>,
595                       SMEM_Real_SGPR_alt_gfx9;
596  let IsGFX9SpecificEncoding = true in
597  def _SGPR_IMM_gfx9 : SMEM_Real_Load_vi <op, ps, SGPR_IMM_Offset>;
598}
599
600class SMEM_Real_Store_Base_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
601  // encoding
602  bits<7> sdata;
603
604  let sdst = ?;
605  let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
606}
607
608class SMEM_Real_Store_vi <bits<8> op, string ps, dag offsets>
609    : SMEM_Real_Store_Base_vi <op, !cast<SM_Pseudo>(ps)> {
610  RegisterClass SrcClass = !cast<SM_Store_Pseudo>(ps).SrcClass;
611  RegisterClass BaseClass = !cast<SM_Store_Pseudo>(ps).BaseClass;
612  let InOperandList = !con((ins SrcClass:$sdata, BaseClass:$sbase),
613                           offsets, (ins CPol:$cpol));
614}
615
616multiclass SM_Real_Stores_vi<bits<8> op, string ps> {
617  def _IMM_vi : SMEM_Real_Store_vi <op, ps#_IMM, (ins smem_offset:$offset)>;
618  def _SGPR_vi : SMEM_Real_Store_vi <op, ps#_SGPR, (ins SReg_32:$soffset)>;
619  def _SGPR_alt_gfx9 : SMEM_Real_Store_vi <op, ps#"_SGPR",
620                                           (ins SReg_32:$soffset)>,
621                       SMEM_Real_SGPR_alt_gfx9;
622  let IsGFX9SpecificEncoding = true in
623  def _SGPR_IMM_gfx9 : SMEM_Real_Store_vi <
624    op, ps#"_SGPR_IMM", (ins SReg_32:$soffset, smem_offset_mod:$offset)>;
625}
626
627multiclass SM_Real_Probe_vi<bits<8> op, string ps> {
628  def _IMM_vi  : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
629  def _SGPR_vi : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
630  def _SGPR_alt_gfx9
631    : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>,
632      SMEM_Real_SGPR_alt_gfx9;
633  let IsGFX9SpecificEncoding = true in
634  def _SGPR_IMM_gfx9
635    : SMEM_Real_Store_Base_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>;
636}
637
638defm S_LOAD_DWORD           : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">;
639defm S_LOAD_DWORDX2         : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">;
640defm S_LOAD_DWORDX4         : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">;
641defm S_LOAD_DWORDX8         : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">;
642defm S_LOAD_DWORDX16        : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">;
643defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">;
644defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">;
645defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">;
646defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">;
647defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">;
648
649defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">;
650defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">;
651defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">;
652
653defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">;
654defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">;
655defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">;
656
657// These instructions use same encoding
658def S_DCACHE_INV_vi         : SMEM_Real_vi <0x20, S_DCACHE_INV>;
659def S_DCACHE_WB_vi          : SMEM_Real_vi <0x21, S_DCACHE_WB>;
660def S_DCACHE_INV_VOL_vi     : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>;
661def S_DCACHE_WB_VOL_vi      : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>;
662def S_MEMTIME_vi            : SMEM_Real_vi <0x24, S_MEMTIME>;
663def S_MEMREALTIME_vi        : SMEM_Real_vi <0x25, S_MEMREALTIME>;
664
665defm S_SCRATCH_LOAD_DWORD    : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">;
666defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">;
667defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">;
668
669defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">;
670defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">;
671defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">;
672
673defm S_ATC_PROBE        : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">;
674defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">;
675
676//===----------------------------------------------------------------------===//
677// GFX9
678//===----------------------------------------------------------------------===//
679
680class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps>
681  : SMEM_Real_vi <op, ps>,
682    AtomicNoRet <!subst("_RTN","",NAME), ps.glc> {
683
684  bits<7> sdata;
685
686  let Constraints = ps.Constraints;
687  let DisableEncoding = ps.DisableEncoding;
688
689  let cpol{CPolBit.GLC} = ps.glc;
690  let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0});
691}
692
693multiclass SM_Real_Atomics_vi<bits<8> op, string ps> {
694  def _IMM_vi       : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;
695  def _SGPR_vi      : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;
696  def _SGPR_alt_gfx9
697    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>,
698      SMEM_Real_SGPR_alt_gfx9;
699  let IsGFX9SpecificEncoding = true in
700  def _SGPR_IMM_gfx9
701    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>;
702  def _IMM_RTN_vi   : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;
703  def _SGPR_RTN_vi  : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;
704  def _SGPR_RTN_alt_gfx9
705    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>,
706      SMEM_Real_SGPR_alt_gfx9;
707  let IsGFX9SpecificEncoding = true in
708  def _SGPR_IMM_RTN_gfx9
709    : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;
710}
711
712defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">;
713defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">;
714defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">;
715defm S_BUFFER_ATOMIC_SUB          : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">;
716defm S_BUFFER_ATOMIC_SMIN         : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">;
717defm S_BUFFER_ATOMIC_UMIN         : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">;
718defm S_BUFFER_ATOMIC_SMAX         : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">;
719defm S_BUFFER_ATOMIC_UMAX         : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">;
720defm S_BUFFER_ATOMIC_AND          : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">;
721defm S_BUFFER_ATOMIC_OR           : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">;
722defm S_BUFFER_ATOMIC_XOR          : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">;
723defm S_BUFFER_ATOMIC_INC          : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">;
724defm S_BUFFER_ATOMIC_DEC          : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">;
725
726defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">;
727defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">;
728defm S_BUFFER_ATOMIC_ADD_X2       : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">;
729defm S_BUFFER_ATOMIC_SUB_X2       : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">;
730defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">;
731defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">;
732defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">;
733defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">;
734defm S_BUFFER_ATOMIC_AND_X2       : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">;
735defm S_BUFFER_ATOMIC_OR_X2        : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">;
736defm S_BUFFER_ATOMIC_XOR_X2       : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">;
737defm S_BUFFER_ATOMIC_INC_X2       : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">;
738defm S_BUFFER_ATOMIC_DEC_X2       : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">;
739
740defm S_ATOMIC_SWAP                : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">;
741defm S_ATOMIC_CMPSWAP             : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">;
742defm S_ATOMIC_ADD                 : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">;
743defm S_ATOMIC_SUB                 : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">;
744defm S_ATOMIC_SMIN                : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">;
745defm S_ATOMIC_UMIN                : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">;
746defm S_ATOMIC_SMAX                : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">;
747defm S_ATOMIC_UMAX                : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">;
748defm S_ATOMIC_AND                 : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">;
749defm S_ATOMIC_OR                  : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">;
750defm S_ATOMIC_XOR                 : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">;
751defm S_ATOMIC_INC                 : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">;
752defm S_ATOMIC_DEC                 : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">;
753
754defm S_ATOMIC_SWAP_X2             : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">;
755defm S_ATOMIC_CMPSWAP_X2          : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">;
756defm S_ATOMIC_ADD_X2              : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">;
757defm S_ATOMIC_SUB_X2              : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">;
758defm S_ATOMIC_SMIN_X2             : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">;
759defm S_ATOMIC_UMIN_X2             : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">;
760defm S_ATOMIC_SMAX_X2             : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">;
761defm S_ATOMIC_UMAX_X2             : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">;
762defm S_ATOMIC_AND_X2              : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">;
763defm S_ATOMIC_OR_X2               : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">;
764defm S_ATOMIC_XOR_X2              : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">;
765defm S_ATOMIC_INC_X2              : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">;
766defm S_ATOMIC_DEC_X2              : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">;
767
768multiclass SM_Real_Discard_vi<bits<8> op, string ps> {
769  def _IMM_vi  : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>;
770  def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>;
771  def _SGPR_alt_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>,
772                       SMEM_Real_SGPR_alt_gfx9;
773  let IsGFX9SpecificEncoding = true in
774  def _SGPR_IMM_gfx9 : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR_IMM)>;
775}
776
777defm S_DCACHE_DISCARD    : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">;
778defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">;
779
780//===----------------------------------------------------------------------===//
781// CI
782//===----------------------------------------------------------------------===//
783
784def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset",
785                                          NamedMatchClass<"SMRDLiteralOffset">> {
786  let OperandType = "OPERAND_IMMEDIATE";
787}
788
789class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> :
790  SM_Real<ps>,
791  Enc64 {
792
793  let AssemblerPredicate = isGFX7Only;
794  let DecoderNamespace = "GFX7";
795  let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, CPol:$cpol);
796
797  let Inst{7-0}   = 0xff;
798  let Inst{8}     = 0;
799  let Inst{14-9}  = sbase{6-1};
800  let Inst{21-15} = sdst{6-0};
801  let Inst{26-22} = op;
802  let Inst{31-27} = 0x18; //encoding
803  let Inst{63-32} = offset{31-0};
804}
805
806def S_LOAD_DWORD_IMM_ci           : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>;
807def S_LOAD_DWORDX2_IMM_ci         : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>;
808def S_LOAD_DWORDX4_IMM_ci         : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>;
809def S_LOAD_DWORDX8_IMM_ci         : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>;
810def S_LOAD_DWORDX16_IMM_ci        : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>;
811def S_BUFFER_LOAD_DWORD_IMM_ci    : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>;
812def S_BUFFER_LOAD_DWORDX2_IMM_ci  : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>;
813def S_BUFFER_LOAD_DWORDX4_IMM_ci  : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>;
814def S_BUFFER_LOAD_DWORDX8_IMM_ci  : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>;
815def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>;
816
817class SMRD_Real_ci <bits<5> op, SM_Pseudo ps>
818  : SM_Real<ps>
819  , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>
820  , Enc32 {
821
822  let AssemblerPredicate = isGFX7Only;
823  let DecoderNamespace = "GFX7";
824
825  let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, !if(ps.has_soffset, soffset, ?));
826  let Inst{8}     = ps.has_offset;
827  let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
828  let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
829  let Inst{26-22} = op;
830  let Inst{31-27} = 0x18; //encoding
831}
832
833def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
834
835//===----------------------------------------------------------------------===//
836// Scalar Memory Patterns
837//===----------------------------------------------------------------------===//
838
839def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]> {
840  let GISelPredicateCode = [{
841    if (!MI.hasOneMemOperand())
842      return false;
843    if (!isInstrUniform(MI))
844      return false;
845
846    // FIXME: We should probably be caching this.
847    SmallVector<GEPInfo, 4> AddrInfo;
848    getAddrModeInfo(MI, MRI, AddrInfo);
849
850    if (hasVgprParts(AddrInfo))
851      return false;
852    return true;
853  }];
854}
855
856def SMRDImm         : ComplexPattern<iPTR, 2, "SelectSMRDImm">;
857def SMRDImm32       : ComplexPattern<iPTR, 2, "SelectSMRDImm32">;
858def SMRDSgpr        : ComplexPattern<iPTR, 2, "SelectSMRDSgpr">;
859def SMRDSgprImm     : ComplexPattern<iPTR, 3, "SelectSMRDSgprImm">;
860def SMRDBufferImm   : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm">;
861def SMRDBufferImm32 : ComplexPattern<iPTR, 1, "SelectSMRDBufferImm32">;
862
863multiclass SMRD_Pattern <string Instr, ValueType vt> {
864
865  // 1. IMM offset
866  def : GCNPat <
867    (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
868    (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))
869  >;
870
871  // 2. 32-bit IMM offset on CI
872  def : GCNPat <
873    (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
874    (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> {
875    let OtherPredicates = [isGFX7Only];
876  }
877
878  // 3. SGPR offset
879  def : GCNPat <
880    (smrd_load (SMRDSgpr i64:$sbase, i32:$soffset)),
881    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $soffset, 0))
882  >;
883
884  // 4. SGPR+IMM offset
885  def : GCNPat <
886    (smrd_load (SMRDSgprImm i64:$sbase, i32:$soffset, i32:$offset)),
887    (vt (!cast<SM_Pseudo>(Instr#"_SGPR_IMM") $sbase, $soffset, $offset, 0))> {
888    let OtherPredicates = [isGFX9Plus];
889  }
890
891  // 5. No offset
892  def : GCNPat <
893    (vt (smrd_load (i64 SReg_64:$sbase))),
894    (vt (!cast<SM_Pseudo>(Instr#"_IMM") i64:$sbase, 0, 0))
895  >;
896}
897
898multiclass SMLoad_Pattern <string Instr, ValueType vt> {
899  // 1. Offset as an immediate
900  def : GCNPat <
901    (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), timm:$cachepolicy),
902    (vt (!cast<SM_Pseudo>(Instr#"_IMM") SReg_128:$sbase, i32imm:$offset, (extract_cpol $cachepolicy)))> {
903    let AddedComplexity = 2;
904  }
905
906  // 2. 32-bit IMM offset on CI
907  def : GCNPat <
908    (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), timm:$cachepolicy)),
909    (!cast<InstSI>(Instr#"_IMM_ci") SReg_128:$sbase, smrd_literal_offset:$offset,
910                                    (extract_cpol $cachepolicy))> {
911    let OtherPredicates = [isGFX7Only];
912    let AddedComplexity = 1;
913  }
914
915  // 3. Offset loaded in an 32bit SGPR
916  def : GCNPat <
917    (SIsbuffer_load v4i32:$sbase, i32:$offset, timm:$cachepolicy),
918    (vt (!cast<SM_Pseudo>(Instr#"_SGPR") SReg_128:$sbase, SReg_32:$offset, (extract_cpol $cachepolicy)))
919  >;
920}
921
922// Global and constant loads can be selected to either MUBUF or SMRD
923// instructions, but SMRD instructions are faster so we want the instruction
924// selector to prefer those.
925let AddedComplexity = 100 in {
926
927foreach vt = Reg32Types.types in {
928defm : SMRD_Pattern <"S_LOAD_DWORD", vt>;
929}
930
931foreach vt = SReg_64.RegTypes in {
932defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>;
933}
934
935foreach vt = SReg_128.RegTypes in {
936defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>;
937}
938
939foreach vt = SReg_256.RegTypes in {
940defm : SMRD_Pattern <"S_LOAD_DWORDX8", vt>;
941}
942
943foreach vt = SReg_512.RegTypes in {
944defm : SMRD_Pattern <"S_LOAD_DWORDX16", vt>;
945}
946
947defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD",     i32>;
948defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2",   v2i32>;
949defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4",   v4i32>;
950defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8",   v8i32>;
951defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16",  v16i32>;
952
953defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD",     f32>;
954defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2",   v2f32>;
955defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4",   v4f32>;
956defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8",   v8f32>;
957defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16",  v16f32>;
958} // End let AddedComplexity = 100
959
960let OtherPredicates = [HasSMemTimeInst] in {
961def : GCNPat <
962  (i64 (readcyclecounter)),
963  (S_MEMTIME)
964>;
965} // let OtherPredicates = [HasSMemTimeInst]
966
967let OtherPredicates = [HasShaderCyclesRegister] in {
968def : GCNPat <
969  (i64 (readcyclecounter)),
970  (REG_SEQUENCE SReg_64,
971    (S_GETREG_B32 getHwRegImm<HWREG.SHADER_CYCLES, 0, -12>.ret), sub0,
972    (S_MOV_B32 (i32 0)), sub1)> {
973  // Prefer this to s_memtime because it has lower and more predictable latency.
974  let AddedComplexity = 1;
975}
976} // let OtherPredicates = [HasShaderCyclesRegister]
977
978//===----------------------------------------------------------------------===//
979// GFX10.
980//===----------------------------------------------------------------------===//
981
982class SMEM_Real_10Plus_common<bits<8> op, SM_Pseudo ps, string opName,
983                              int subtarget, RegisterWithSubRegs sgpr_null> :
984    SM_Real<ps, opName>, SIMCInstr<ps.PseudoInstr, subtarget>, Enc64 {
985  let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
986  let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
987  let Inst{25-18} = op;
988  let Inst{31-26} = 0x3d;
989  // There are SMEM instructions that do not employ any of the offset
990  // fields, in which case we need them to remain undefined.
991  let Inst{52-32} = !if(ps.has_offset, offset{20-0}, !if(ps.has_soffset, 0, ?));
992  let Inst{63-57} = !if(ps.has_soffset, soffset{6-0},
993                        !if(ps.has_offset, sgpr_null.HWEncoding{6-0}, ?));
994}
995
996class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps>
997    : SMEM_Real_10Plus_common<op, ps, ps.Mnemonic, SIEncodingFamily.GFX10,
998                              SGPR_NULL_gfxpre11> {
999  let AssemblerPredicate = isGFX10Only;
1000  let DecoderNamespace = "GFX10";
1001  let Inst{14}    = !if(ps.has_dlc, cpol{CPolBit.DLC}, ?);
1002  let Inst{16}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ?);
1003}
1004
1005class SMEM_Real_Load_gfx10<bits<8> op, string ps, OffsetMode offsets>
1006    : SMEM_Real_gfx10<op, !cast<SM_Pseudo>(ps # offsets.Variant)> {
1007  RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass;
1008  let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol));
1009}
1010
1011multiclass SM_Real_Loads_gfx10<bits<8> op, string ps> {
1012  def _IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps, IMM_Offset>;
1013  def _SGPR_gfx10 : SMEM_Real_Load_gfx10<op, ps, SGPR_Offset>;
1014  def _SGPR_IMM_gfx10 : SMEM_Real_Load_gfx10<op, ps, SGPR_IMM_Offset>;
1015}
1016
1017class SMEM_Real_Store_gfx10<bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx10<op, ps> {
1018  bits<7> sdata;
1019
1020  let sdst = ?;
1021  let Inst{12-6} = !if(ps.has_sdst, sdata{6-0}, ?);
1022}
1023
1024multiclass SM_Real_Stores_gfx10<bits<8> op, string ps,
1025                                SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
1026                                SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
1027  def _IMM_gfx10 : SMEM_Real_Store_gfx10 <op, immPs> {
1028    let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smem_offset:$offset, CPol:$cpol);
1029  }
1030
1031  def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, sgprPs> {
1032    let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$soffset, CPol:$cpol);
1033  }
1034
1035  def _SGPR_IMM_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Store_Pseudo>(ps#_SGPR_IMM)> {
1036    let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase,
1037                             SReg_32:$soffset, smem_offset_mod:$offset, CPol:$cpol);
1038  }
1039}
1040
1041defm S_LOAD_DWORD            : SM_Real_Loads_gfx10<0x000, "S_LOAD_DWORD">;
1042defm S_LOAD_DWORDX2          : SM_Real_Loads_gfx10<0x001, "S_LOAD_DWORDX2">;
1043defm S_LOAD_DWORDX4          : SM_Real_Loads_gfx10<0x002, "S_LOAD_DWORDX4">;
1044defm S_LOAD_DWORDX8          : SM_Real_Loads_gfx10<0x003, "S_LOAD_DWORDX8">;
1045defm S_LOAD_DWORDX16         : SM_Real_Loads_gfx10<0x004, "S_LOAD_DWORDX16">;
1046
1047let SubtargetPredicate = HasScalarFlatScratchInsts in {
1048defm S_SCRATCH_LOAD_DWORD    : SM_Real_Loads_gfx10<0x005, "S_SCRATCH_LOAD_DWORD">;
1049defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_gfx10<0x006, "S_SCRATCH_LOAD_DWORDX2">;
1050defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_gfx10<0x007, "S_SCRATCH_LOAD_DWORDX4">;
1051} // End SubtargetPredicate = HasScalarFlatScratchInsts
1052
1053defm S_BUFFER_LOAD_DWORD     : SM_Real_Loads_gfx10<0x008, "S_BUFFER_LOAD_DWORD">;
1054defm S_BUFFER_LOAD_DWORDX2   : SM_Real_Loads_gfx10<0x009, "S_BUFFER_LOAD_DWORDX2">;
1055defm S_BUFFER_LOAD_DWORDX4   : SM_Real_Loads_gfx10<0x00a, "S_BUFFER_LOAD_DWORDX4">;
1056defm S_BUFFER_LOAD_DWORDX8   : SM_Real_Loads_gfx10<0x00b, "S_BUFFER_LOAD_DWORDX8">;
1057defm S_BUFFER_LOAD_DWORDX16  : SM_Real_Loads_gfx10<0x00c, "S_BUFFER_LOAD_DWORDX16">;
1058
1059let SubtargetPredicate = HasScalarStores in {
1060defm S_STORE_DWORD           : SM_Real_Stores_gfx10<0x010, "S_STORE_DWORD">;
1061defm S_STORE_DWORDX2         : SM_Real_Stores_gfx10<0x011, "S_STORE_DWORDX2">;
1062defm S_STORE_DWORDX4         : SM_Real_Stores_gfx10<0x012, "S_STORE_DWORDX4">;
1063let OtherPredicates = [HasScalarFlatScratchInsts] in {
1064defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_gfx10<0x015, "S_SCRATCH_STORE_DWORD">;
1065defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_gfx10<0x016, "S_SCRATCH_STORE_DWORDX2">;
1066defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_gfx10<0x017, "S_SCRATCH_STORE_DWORDX4">;
1067} // End OtherPredicates = [HasScalarFlatScratchInsts]
1068defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_gfx10<0x018, "S_BUFFER_STORE_DWORD">;
1069defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_gfx10<0x019, "S_BUFFER_STORE_DWORDX2">;
1070defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_gfx10<0x01a, "S_BUFFER_STORE_DWORDX4">;
1071} // End SubtargetPredicate = HasScalarStores
1072
1073def S_MEMREALTIME_gfx10              : SMEM_Real_gfx10<0x025, S_MEMREALTIME>;
1074def S_MEMTIME_gfx10                  : SMEM_Real_gfx10<0x024, S_MEMTIME>;
1075def S_GL1_INV_gfx10                  : SMEM_Real_gfx10<0x01f, S_GL1_INV>;
1076def S_GET_WAVEID_IN_WORKGROUP_gfx10  : SMEM_Real_gfx10<0x02a, S_GET_WAVEID_IN_WORKGROUP>;
1077def S_DCACHE_INV_gfx10               : SMEM_Real_gfx10<0x020, S_DCACHE_INV>;
1078
1079let SubtargetPredicate = HasScalarStores in {
1080def S_DCACHE_WB_gfx10                : SMEM_Real_gfx10<0x021, S_DCACHE_WB>;
1081} // End SubtargetPredicate = HasScalarStores
1082
1083multiclass SM_Real_Probe_gfx10<bits<8> op, string ps> {
1084  def _IMM_gfx10  : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;
1085  def _SGPR_gfx10 : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;
1086  def _SGPR_IMM_gfx10
1087    : SMEM_Real_Store_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>;
1088}
1089
1090defm S_ATC_PROBE        : SM_Real_Probe_gfx10 <0x26, "S_ATC_PROBE">;
1091defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx10 <0x27, "S_ATC_PROBE_BUFFER">;
1092
1093class SMEM_Atomic_Real_gfx10 <bits<8> op, SM_Atomic_Pseudo ps>
1094  : SMEM_Real_gfx10 <op, ps>,
1095    AtomicNoRet <!subst("_RTN","",NAME), ps.glc> {
1096
1097  bits<7> sdata;
1098
1099  let Constraints = ps.Constraints;
1100  let DisableEncoding = ps.DisableEncoding;
1101
1102  let cpol{CPolBit.GLC} = ps.glc;
1103
1104  let Inst{14} = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0);
1105  let Inst{12-6} = !if(ps.glc, sdst{6-0}, sdata{6-0});
1106}
1107
1108multiclass SM_Real_Atomics_gfx10<bits<8> op, string ps> {
1109  def _IMM_gfx10       : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;
1110  def _SGPR_gfx10      : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;
1111  def _SGPR_IMM_gfx10  : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM)>;
1112  def _IMM_RTN_gfx10   : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;
1113  def _SGPR_RTN_gfx10  : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;
1114  def _SGPR_IMM_RTN_gfx10 : SMEM_Atomic_Real_gfx10 <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_IMM_RTN)>;
1115}
1116
1117let SubtargetPredicate = HasScalarAtomics in {
1118
1119defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_gfx10 <0x40, "S_BUFFER_ATOMIC_SWAP">;
1120defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_gfx10 <0x41, "S_BUFFER_ATOMIC_CMPSWAP">;
1121defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_gfx10 <0x42, "S_BUFFER_ATOMIC_ADD">;
1122defm S_BUFFER_ATOMIC_SUB          : SM_Real_Atomics_gfx10 <0x43, "S_BUFFER_ATOMIC_SUB">;
1123defm S_BUFFER_ATOMIC_SMIN         : SM_Real_Atomics_gfx10 <0x44, "S_BUFFER_ATOMIC_SMIN">;
1124defm S_BUFFER_ATOMIC_UMIN         : SM_Real_Atomics_gfx10 <0x45, "S_BUFFER_ATOMIC_UMIN">;
1125defm S_BUFFER_ATOMIC_SMAX         : SM_Real_Atomics_gfx10 <0x46, "S_BUFFER_ATOMIC_SMAX">;
1126defm S_BUFFER_ATOMIC_UMAX         : SM_Real_Atomics_gfx10 <0x47, "S_BUFFER_ATOMIC_UMAX">;
1127defm S_BUFFER_ATOMIC_AND          : SM_Real_Atomics_gfx10 <0x48, "S_BUFFER_ATOMIC_AND">;
1128defm S_BUFFER_ATOMIC_OR           : SM_Real_Atomics_gfx10 <0x49, "S_BUFFER_ATOMIC_OR">;
1129defm S_BUFFER_ATOMIC_XOR          : SM_Real_Atomics_gfx10 <0x4a, "S_BUFFER_ATOMIC_XOR">;
1130defm S_BUFFER_ATOMIC_INC          : SM_Real_Atomics_gfx10 <0x4b, "S_BUFFER_ATOMIC_INC">;
1131defm S_BUFFER_ATOMIC_DEC          : SM_Real_Atomics_gfx10 <0x4c, "S_BUFFER_ATOMIC_DEC">;
1132
1133defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Real_Atomics_gfx10 <0x60, "S_BUFFER_ATOMIC_SWAP_X2">;
1134defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Real_Atomics_gfx10 <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">;
1135defm S_BUFFER_ATOMIC_ADD_X2       : SM_Real_Atomics_gfx10 <0x62, "S_BUFFER_ATOMIC_ADD_X2">;
1136defm S_BUFFER_ATOMIC_SUB_X2       : SM_Real_Atomics_gfx10 <0x63, "S_BUFFER_ATOMIC_SUB_X2">;
1137defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Real_Atomics_gfx10 <0x64, "S_BUFFER_ATOMIC_SMIN_X2">;
1138defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Real_Atomics_gfx10 <0x65, "S_BUFFER_ATOMIC_UMIN_X2">;
1139defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Real_Atomics_gfx10 <0x66, "S_BUFFER_ATOMIC_SMAX_X2">;
1140defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Real_Atomics_gfx10 <0x67, "S_BUFFER_ATOMIC_UMAX_X2">;
1141defm S_BUFFER_ATOMIC_AND_X2       : SM_Real_Atomics_gfx10 <0x68, "S_BUFFER_ATOMIC_AND_X2">;
1142defm S_BUFFER_ATOMIC_OR_X2        : SM_Real_Atomics_gfx10 <0x69, "S_BUFFER_ATOMIC_OR_X2">;
1143defm S_BUFFER_ATOMIC_XOR_X2       : SM_Real_Atomics_gfx10 <0x6a, "S_BUFFER_ATOMIC_XOR_X2">;
1144defm S_BUFFER_ATOMIC_INC_X2       : SM_Real_Atomics_gfx10 <0x6b, "S_BUFFER_ATOMIC_INC_X2">;
1145defm S_BUFFER_ATOMIC_DEC_X2       : SM_Real_Atomics_gfx10 <0x6c, "S_BUFFER_ATOMIC_DEC_X2">;
1146
1147defm S_ATOMIC_SWAP                : SM_Real_Atomics_gfx10 <0x80, "S_ATOMIC_SWAP">;
1148defm S_ATOMIC_CMPSWAP             : SM_Real_Atomics_gfx10 <0x81, "S_ATOMIC_CMPSWAP">;
1149defm S_ATOMIC_ADD                 : SM_Real_Atomics_gfx10 <0x82, "S_ATOMIC_ADD">;
1150defm S_ATOMIC_SUB                 : SM_Real_Atomics_gfx10 <0x83, "S_ATOMIC_SUB">;
1151defm S_ATOMIC_SMIN                : SM_Real_Atomics_gfx10 <0x84, "S_ATOMIC_SMIN">;
1152defm S_ATOMIC_UMIN                : SM_Real_Atomics_gfx10 <0x85, "S_ATOMIC_UMIN">;
1153defm S_ATOMIC_SMAX                : SM_Real_Atomics_gfx10 <0x86, "S_ATOMIC_SMAX">;
1154defm S_ATOMIC_UMAX                : SM_Real_Atomics_gfx10 <0x87, "S_ATOMIC_UMAX">;
1155defm S_ATOMIC_AND                 : SM_Real_Atomics_gfx10 <0x88, "S_ATOMIC_AND">;
1156defm S_ATOMIC_OR                  : SM_Real_Atomics_gfx10 <0x89, "S_ATOMIC_OR">;
1157defm S_ATOMIC_XOR                 : SM_Real_Atomics_gfx10 <0x8a, "S_ATOMIC_XOR">;
1158defm S_ATOMIC_INC                 : SM_Real_Atomics_gfx10 <0x8b, "S_ATOMIC_INC">;
1159defm S_ATOMIC_DEC                 : SM_Real_Atomics_gfx10 <0x8c, "S_ATOMIC_DEC">;
1160
1161defm S_ATOMIC_SWAP_X2             : SM_Real_Atomics_gfx10 <0xa0, "S_ATOMIC_SWAP_X2">;
1162defm S_ATOMIC_CMPSWAP_X2          : SM_Real_Atomics_gfx10 <0xa1, "S_ATOMIC_CMPSWAP_X2">;
1163defm S_ATOMIC_ADD_X2              : SM_Real_Atomics_gfx10 <0xa2, "S_ATOMIC_ADD_X2">;
1164defm S_ATOMIC_SUB_X2              : SM_Real_Atomics_gfx10 <0xa3, "S_ATOMIC_SUB_X2">;
1165defm S_ATOMIC_SMIN_X2             : SM_Real_Atomics_gfx10 <0xa4, "S_ATOMIC_SMIN_X2">;
1166defm S_ATOMIC_UMIN_X2             : SM_Real_Atomics_gfx10 <0xa5, "S_ATOMIC_UMIN_X2">;
1167defm S_ATOMIC_SMAX_X2             : SM_Real_Atomics_gfx10 <0xa6, "S_ATOMIC_SMAX_X2">;
1168defm S_ATOMIC_UMAX_X2             : SM_Real_Atomics_gfx10 <0xa7, "S_ATOMIC_UMAX_X2">;
1169defm S_ATOMIC_AND_X2              : SM_Real_Atomics_gfx10 <0xa8, "S_ATOMIC_AND_X2">;
1170defm S_ATOMIC_OR_X2               : SM_Real_Atomics_gfx10 <0xa9, "S_ATOMIC_OR_X2">;
1171defm S_ATOMIC_XOR_X2              : SM_Real_Atomics_gfx10 <0xaa, "S_ATOMIC_XOR_X2">;
1172defm S_ATOMIC_INC_X2              : SM_Real_Atomics_gfx10 <0xab, "S_ATOMIC_INC_X2">;
1173defm S_ATOMIC_DEC_X2              : SM_Real_Atomics_gfx10 <0xac, "S_ATOMIC_DEC_X2">;
1174
1175multiclass SM_Real_Discard_gfx10<bits<8> op, string ps> {
1176  def _IMM_gfx10  : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_IMM)>;
1177  def _SGPR_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR)>;
1178  def _SGPR_IMM_gfx10 : SMEM_Real_gfx10 <op, !cast<SM_Pseudo>(ps#_SGPR_IMM)>;
1179}
1180
1181defm S_DCACHE_DISCARD    : SM_Real_Discard_gfx10 <0x28, "S_DCACHE_DISCARD">;
1182defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_gfx10 <0x29, "S_DCACHE_DISCARD_X2">;
1183
1184} // End SubtargetPredicate = HasScalarAtomics
1185
1186def SMInfoTable : GenericTable {
1187  let FilterClass = "SM_Real";
1188  let CppTypeName = "SMInfo";
1189  let Fields = ["Opcode", "is_buffer"];
1190
1191  let PrimaryKey = ["Opcode"];
1192  let PrimaryKeyName = "getSMEMOpcodeHelper";
1193}
1194
1195//===----------------------------------------------------------------------===//
1196// GFX11.
1197//===----------------------------------------------------------------------===//
1198
1199class SMEM_Real_gfx11<bits<8> op, SM_Pseudo ps, string opName = ps.Mnemonic> :
1200    SMEM_Real_10Plus_common<op, ps, opName, SIEncodingFamily.GFX11,
1201                            SGPR_NULL_gfx11plus> {
1202  let AssemblerPredicate = isGFX11Plus;
1203  let DecoderNamespace = "GFX11";
1204  let Inst{13}    = !if(ps.has_dlc, cpol{CPolBit.DLC}, 0);
1205  let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, 0);
1206}
1207
1208class SMEM_Real_Load_gfx11<bits<8> op, string ps, string opName, OffsetMode offsets> :
1209    SMEM_Real_gfx11<op, !cast<SM_Pseudo>(ps # offsets.Variant), opName> {
1210  RegisterClass BaseClass = !cast<SM_Load_Pseudo>(ps # offsets.Variant).BaseClass;
1211  let InOperandList = !con((ins BaseClass:$sbase), offsets.Ins, (ins CPol:$cpol));
1212}
1213
1214multiclass SM_Real_Loads_gfx11<bits<8> op, string ps, string opName> {
1215  def _IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps, opName, IMM_Offset>;
1216  def _SGPR_gfx11 : SMEM_Real_Load_gfx11<op, ps, opName, SGPR_Offset>;
1217  def _SGPR_IMM_gfx11 : SMEM_Real_Load_gfx11<op, ps, opName, SGPR_IMM_Offset>;
1218  def : MnemonicAlias<!cast<SM_Pseudo>(ps#"_IMM").Mnemonic, opName>,
1219                      Requires<[isGFX11Plus]>;
1220}
1221
1222defm S_LOAD_B32  : SM_Real_Loads_gfx11<0x000, "S_LOAD_DWORD", "s_load_b32">;
1223defm S_LOAD_B64  : SM_Real_Loads_gfx11<0x001, "S_LOAD_DWORDX2", "s_load_b64">;
1224defm S_LOAD_B128 : SM_Real_Loads_gfx11<0x002, "S_LOAD_DWORDX4", "s_load_b128">;
1225defm S_LOAD_B256 : SM_Real_Loads_gfx11<0x003, "S_LOAD_DWORDX8", "s_load_b256">;
1226defm S_LOAD_B512 : SM_Real_Loads_gfx11<0x004, "S_LOAD_DWORDX16", "s_load_b512">;
1227
1228defm S_BUFFER_LOAD_B32  : SM_Real_Loads_gfx11<0x008, "S_BUFFER_LOAD_DWORD", "s_buffer_load_b32">;
1229defm S_BUFFER_LOAD_B64  : SM_Real_Loads_gfx11<0x009, "S_BUFFER_LOAD_DWORDX2", "s_buffer_load_b64">;
1230defm S_BUFFER_LOAD_B128 : SM_Real_Loads_gfx11<0x00a, "S_BUFFER_LOAD_DWORDX4", "s_buffer_load_b128">;
1231defm S_BUFFER_LOAD_B256 : SM_Real_Loads_gfx11<0x00b, "S_BUFFER_LOAD_DWORDX8", "s_buffer_load_b256">;
1232defm S_BUFFER_LOAD_B512 : SM_Real_Loads_gfx11<0x00c, "S_BUFFER_LOAD_DWORDX16", "s_buffer_load_b512">;
1233
1234def S_GL1_INV_gfx11    : SMEM_Real_gfx11<0x020, S_GL1_INV>;
1235def S_DCACHE_INV_gfx11 : SMEM_Real_gfx11<0x021, S_DCACHE_INV>;
1236
1237class SMEM_Real_Store_gfx11 <bits<8> op, SM_Pseudo ps> : SMEM_Real_gfx11<op, ps> {
1238  // encoding
1239  bits<7> sdata;
1240
1241  let sdst = ?;
1242  let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
1243}
1244
1245multiclass SM_Real_Probe_gfx11<bits<8> op, string ps> {
1246  def _IMM_gfx11  : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
1247  def _SGPR_gfx11 : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
1248  def _SGPR_IMM_gfx11
1249    : SMEM_Real_Store_gfx11 <op, !cast<SM_Probe_Pseudo>(ps#_SGPR_IMM)>;
1250}
1251
1252defm S_ATC_PROBE        : SM_Real_Probe_gfx11 <0x22, "S_ATC_PROBE">;
1253defm S_ATC_PROBE_BUFFER : SM_Real_Probe_gfx11 <0x23, "S_ATC_PROBE_BUFFER">;
1254