xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/DSInstructions.td (revision f126890ac5386406dadf7c4cfa9566cbb56537c5) 
1//===-- DSInstructions.td - DS Instruction Definitions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
10  InstSI <outs, ins, "", pattern>,
11  SIMCInstr <opName, SIEncodingFamily.NONE> {
12
13  let LGKM_CNT = 1;
14  let DS = 1;
15  let Size = 8;
16  let UseNamedOperandTable = 1;
17
18  // Most instruction load and store data, so set this as the default.
19  let mayLoad = 1;
20  let mayStore = 1;
21  let maybeAtomic = 1;
22
23  let hasSideEffects = 0;
24  let SchedRW = [WriteLDS];
25
26  let isPseudo = 1;
27  let isCodeGenOnly = 1;
28
29  string Mnemonic = opName;
30  string AsmOperands = asmOps;
31
32  // Well these bits a kind of hack because it would be more natural
33  // to test "outs" and "ins" dags for the presence of particular operands
34  bits<1> has_vdst = 1;
35  bits<1> has_addr = 1;
36  bits<1> has_data0 = 1;
37  bits<1> has_data1 = 1;
38
39  bits<1> has_gws_data0 = 0; // data0 is encoded as addr
40
41  bits<1> has_offset  = 1; // has "offset" that should be split to offset0,1
42  bits<1> has_offset0 = 1;
43  bits<1> has_offset1 = 1;
44
45  bits<1> has_gds = 1;
46  bits<1> gdsValue = 0; // if has_gds == 0 set gds to this value
47
48  bits<1> has_m0_read = 1;
49
50  let Uses = !if(has_m0_read, [M0, EXEC], [EXEC]);
51}
52
53class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
54  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands>,
55  Enc64 {
56
57  let isPseudo = 0;
58  let isCodeGenOnly = 0;
59  let LGKM_CNT = 1;
60  let DS = 1;
61  let UseNamedOperandTable = 1;
62
63  // copy relevant pseudo op flags
64  let SubtargetPredicate = ps.SubtargetPredicate;
65  let OtherPredicates    = ps.OtherPredicates;
66  let SchedRW            = ps.SchedRW;
67  let mayLoad            = ps.mayLoad;
68  let mayStore           = ps.mayStore;
69  let IsAtomicRet        = ps.IsAtomicRet;
70  let IsAtomicNoRet      = ps.IsAtomicNoRet;
71
72  let Constraints = ps.Constraints;
73  let DisableEncoding = ps.DisableEncoding;
74
75  // encoding fields
76  bits<10> vdst;
77  bits<1> gds;
78  bits<8> addr;
79  bits<10> data0;
80  bits<10> data1;
81  bits<8> offset0;
82  bits<8> offset1;
83
84  bits<16> offset;
85  let offset0 = !if(ps.has_offset, offset{7-0}, ?);
86  let offset1 = !if(ps.has_offset, offset{15-8}, ?);
87
88  bits<1> acc = !if(ps.has_vdst, vdst{9},
89                    !if(!or(ps.has_data0, ps.has_gws_data0), data0{9}, 0));
90}
91
92// DS Pseudo instructions
93
94class DS_0A1D_NORET<string opName, RegisterClass rc = VGPR_32>
95: DS_Pseudo<opName,
96  (outs),
97  (ins getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
98  " $data0$offset$gds"> {
99
100  let has_addr = 0;
101  let has_data1 = 0;
102  let has_vdst = 0;
103}
104
105class DS_1A1D_NORET<string opName, RegisterClass rc = VGPR_32>
106: DS_Pseudo<opName,
107  (outs),
108  (ins VGPR_32:$addr, getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
109  " $addr, $data0$offset$gds"> {
110
111  let has_data1 = 0;
112  let has_vdst = 0;
113  let IsAtomicNoRet = 1;
114}
115
116multiclass DS_1A1D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
117  def "" : DS_1A1D_NORET<opName, rc>,
118           AtomicNoRet<opName, 0>;
119
120  let has_m0_read = 0 in {
121    def _gfx9 : DS_1A1D_NORET<opName, rc>,
122                AtomicNoRet<opName#"_gfx9", 0>;
123  }
124}
125
126multiclass DS_1A1D_NORET_mc_gfx9<string opName, RegisterClass rc = VGPR_32> {
127  let has_m0_read = 0 in {
128    def "" : DS_1A1D_NORET<opName, rc>,
129                AtomicNoRet<opName, 0>;
130  }
131}
132
133class DS_1A2D_NORET<string opName, RegisterClass rc = VGPR_32,
134                    RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
135: DS_Pseudo<opName,
136  (outs),
137  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1, offset:$offset, gds:$gds),
138  " $addr, $data0, $data1$offset$gds"> {
139
140  let has_vdst = 0;
141  let IsAtomicNoRet = 1;
142}
143
144multiclass DS_1A2D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
145  def "" : DS_1A2D_NORET<opName, rc>,
146           AtomicNoRet<opName, 0>;
147
148  let has_m0_read = 0 in {
149    def _gfx9 : DS_1A2D_NORET<opName, rc>,
150                AtomicNoRet<opName#"_gfx9", 0>;
151  }
152}
153
154class DS_1A2D_Off8_NORET <string opName, RegisterClass rc = VGPR_32,
155                          RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
156: DS_Pseudo<opName,
157  (outs),
158  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1,
159       offset0:$offset0, offset1:$offset1, gds:$gds),
160  " $addr, $data0, $data1$offset0$offset1$gds"> {
161
162  let has_vdst = 0;
163  let has_offset = 0;
164}
165
166multiclass DS_1A2D_Off8_NORET_mc <string opName, RegisterClass rc = VGPR_32> {
167  def "" : DS_1A2D_Off8_NORET<opName, rc>;
168
169  let has_m0_read = 0 in {
170    def _gfx9 : DS_1A2D_Off8_NORET<opName, rc>;
171  }
172}
173
174class DS_0A1D_RET_GDS<string opName, RegisterClass rc = VGPR_32, RegisterClass src = rc,
175                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
176                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
177: DS_Pseudo<opName,
178  (outs dst_op:$vdst),
179  (ins src_op:$data0, offset:$offset),
180  " $vdst, $data0$offset gds"> {
181
182  let has_addr = 0;
183  let has_data1 = 0;
184  let has_gds = 0;
185  let gdsValue = 1;
186  let hasSideEffects = 1;
187}
188
189class DS_1A1D_RET <string opName, RegisterClass rc = VGPR_32,
190                  RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
191: DS_Pseudo<opName,
192  (outs data_op:$vdst),
193  (ins VGPR_32:$addr, data_op:$data0, offset:$offset, gds:$gds),
194  " $vdst, $addr, $data0$offset$gds"> {
195
196  let hasPostISelHook = 1;
197  let has_data1 = 0;
198  let IsAtomicRet = 1;
199}
200
201multiclass DS_1A1D_RET_mc <string opName, RegisterClass rc = VGPR_32,
202                           string NoRetOp = ""> {
203  def "" : DS_1A1D_RET<opName, rc>,
204    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
205
206  let has_m0_read = 0 in {
207    def _gfx9 : DS_1A1D_RET<opName, rc>,
208      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp#"_gfx9"),
209                  !ne(NoRetOp, "")>;
210  }
211}
212
213multiclass DS_1A1D_RET_mc_gfx9 <string opName, RegisterClass rc = VGPR_32,
214                                string NoRetOp = ""> {
215  let has_m0_read = 0 in {
216    def "" : DS_1A1D_RET<opName, rc>,
217      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp),
218                  !ne(NoRetOp, "")>;
219  }
220}
221
222class DS_1A2D_RET<string opName,
223                  RegisterClass rc = VGPR_32,
224                  RegisterClass src = rc,
225                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
226                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
227: DS_Pseudo<opName,
228  (outs dst_op:$vdst),
229  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset:$offset, gds:$gds),
230  " $vdst, $addr, $data0, $data1$offset$gds"> {
231
232  let hasPostISelHook = 1;
233  let IsAtomicRet = 1;
234}
235
236multiclass DS_1A2D_RET_mc<string opName,
237                          RegisterClass rc = VGPR_32,
238                          string NoRetOp = "",
239                          RegisterClass src = rc> {
240  def "" : DS_1A2D_RET<opName, rc, src>,
241    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
242
243  let has_m0_read = 0 in {
244    def _gfx9 : DS_1A2D_RET<opName, rc, src>,
245      AtomicNoRet<NoRetOp#"_gfx9", !ne(NoRetOp, "")>;
246  }
247}
248
249class DS_1A2D_Off8_RET<string opName,
250                       RegisterClass rc = VGPR_32,
251                       RegisterClass src = rc,
252                       RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
253                       RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
254: DS_Pseudo<opName,
255  (outs dst_op:$vdst),
256  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset0:$offset0, offset1:$offset1, gds:$gds),
257  " $vdst, $addr, $data0, $data1$offset0$offset1$gds"> {
258
259  let has_offset = 0;
260  let hasPostISelHook = 1;
261}
262
263multiclass DS_1A2D_Off8_RET_mc<string opName,
264                               RegisterClass rc = VGPR_32,
265                               RegisterClass src = rc> {
266  def "" : DS_1A2D_Off8_RET<opName, rc, src>;
267
268  let has_m0_read = 0 in {
269    def _gfx9 : DS_1A2D_Off8_RET<opName, rc, src>;
270  }
271}
272
273class DS_BVH_STACK<string opName>
274: DS_Pseudo<opName,
275  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst, VGPR_32:$addr),
276  (ins VGPR_32:$addr_in, getLdStRegisterOperand<VGPR_32>.ret:$data0, VReg_128:$data1, offset:$offset),
277  " $vdst, $addr, $data0, $data1$offset"> {
278  let Constraints = "$addr = $addr_in";
279  let DisableEncoding = "$addr_in";
280  let has_gds = 0;
281  let gdsValue = 0;
282  // TODO: Use MMOs in the LDS address space instead of hasSideEffects = 1.
283  let hasSideEffects = 1;
284  let SchedRW = [WriteLDS, WriteLDS];
285}
286
287class DS_1A_RET<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset,
288                RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
289: DS_Pseudo<opName,
290  (outs data_op:$vdst),
291  !if(HasTiedOutput,
292    (ins VGPR_32:$addr, ofs:$offset, gds:$gds, data_op:$vdst_in),
293    (ins VGPR_32:$addr, ofs:$offset, gds:$gds)),
294  " $vdst, $addr$offset$gds"> {
295  let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");
296  let DisableEncoding = !if(HasTiedOutput, "$vdst_in", "");
297  let has_data0 = 0;
298  let has_data1 = 0;
299}
300
301multiclass DS_1A_RET_mc<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset> {
302  def "" : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
303
304  let has_m0_read = 0 in {
305    def _gfx9 : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
306  }
307}
308
309class DS_1A_RET_Tied<string opName, RegisterClass rc = VGPR_32> :
310  DS_1A_RET<opName, rc, 1>;
311
312class DS_1A_Off8_RET <string opName, RegisterClass rc = VGPR_32>
313: DS_Pseudo<opName,
314  (outs getLdStRegisterOperand<rc>.ret:$vdst),
315  (ins VGPR_32:$addr, offset0:$offset0, offset1:$offset1, gds:$gds),
316  " $vdst, $addr$offset0$offset1$gds"> {
317
318  let has_offset = 0;
319  let has_data0 = 0;
320  let has_data1 = 0;
321}
322
323multiclass DS_1A_Off8_RET_mc <string opName, RegisterClass rc = VGPR_32> {
324  def "" : DS_1A_Off8_RET<opName, rc>;
325
326  let has_m0_read = 0 in {
327    def _gfx9 : DS_1A_Off8_RET<opName, rc>;
328  }
329}
330
331class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,
332  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
333  (ins VGPR_32:$addr, offset:$offset),
334  " $vdst, $addr$offset gds"> {
335
336  let has_data0 = 0;
337  let has_data1 = 0;
338  let has_gds = 0;
339  let gdsValue = 1;
340}
341
342class DS_0A_RET <string opName> : DS_Pseudo<opName,
343  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
344  (ins offset:$offset, gds:$gds),
345  " $vdst$offset$gds"> {
346
347  let mayLoad = 1;
348  let mayStore = 1;
349
350  let has_addr = 0;
351  let has_data0 = 0;
352  let has_data1 = 0;
353}
354
355class DS_1A <string opName> : DS_Pseudo<opName,
356  (outs),
357  (ins VGPR_32:$addr, offset:$offset, gds:$gds),
358  " $addr$offset$gds"> {
359
360  let mayLoad = 1;
361  let mayStore = 1;
362
363  let has_vdst = 0;
364  let has_data0 = 0;
365  let has_data1 = 0;
366}
367
368multiclass DS_1A_mc <string opName> {
369  def "" : DS_1A<opName>;
370
371  let has_m0_read = 0 in {
372    def _gfx9 : DS_1A<opName>;
373  }
374}
375
376
377class DS_GWS <string opName, dag ins, string asmOps>
378: DS_Pseudo<opName, (outs), ins, asmOps> {
379
380  let has_vdst  = 0;
381  let has_addr  = 0;
382  let has_data0 = 0;
383  let has_data1 = 0;
384
385  let has_gds   = 0;
386  let gdsValue  = 1;
387}
388
389class DS_GWS_0D <string opName>
390: DS_GWS<opName,
391  (ins offset:$offset), "$offset gds"> {
392  let hasSideEffects = 1;
393}
394
395class DS_GWS_1D <string opName>
396: DS_GWS<opName,
397  (ins getLdStRegisterOperand<VGPR_32>.ret:$data0, offset:$offset),
398  " $data0$offset gds"> {
399
400  let has_gws_data0 = 1;
401  let hasSideEffects = 1;
402}
403
404class DS_VOID <string opName> : DS_Pseudo<opName,
405  (outs), (ins), ""> {
406  let mayLoad = 0;
407  let mayStore = 0;
408  let hasSideEffects = 1;
409  let UseNamedOperandTable = 0;
410
411  let has_vdst = 0;
412  let has_addr = 0;
413  let has_data0 = 0;
414  let has_data1 = 0;
415  let has_offset = 0;
416  let has_offset0 = 0;
417  let has_offset1 = 0;
418  let has_gds = 0;
419}
420
421class DS_1A1D_PERMUTE <string opName, SDPatternOperator node = null_frag,
422                       RegisterOperand data_op = getLdStRegisterOperand<VGPR_32>.ret>
423: DS_Pseudo<opName,
424  (outs data_op:$vdst),
425  (ins VGPR_32:$addr, data_op:$data0, offset:$offset),
426  " $vdst, $addr, $data0$offset",
427  [(set i32:$vdst,
428   (node (DS1Addr1Offset i32:$addr, i32:$offset), i32:$data0))] > {
429
430  let mayLoad = 0;
431  let mayStore = 0;
432  let isConvergent = 1;
433
434  let has_data1 = 0;
435  let has_gds = 0;
436}
437
438defm DS_ADD_U32       : DS_1A1D_NORET_mc<"ds_add_u32">;
439defm DS_SUB_U32       : DS_1A1D_NORET_mc<"ds_sub_u32">;
440defm DS_RSUB_U32      : DS_1A1D_NORET_mc<"ds_rsub_u32">;
441defm DS_INC_U32       : DS_1A1D_NORET_mc<"ds_inc_u32">;
442defm DS_DEC_U32       : DS_1A1D_NORET_mc<"ds_dec_u32">;
443defm DS_MIN_I32       : DS_1A1D_NORET_mc<"ds_min_i32">;
444defm DS_MAX_I32       : DS_1A1D_NORET_mc<"ds_max_i32">;
445defm DS_MIN_U32       : DS_1A1D_NORET_mc<"ds_min_u32">;
446defm DS_MAX_U32       : DS_1A1D_NORET_mc<"ds_max_u32">;
447defm DS_AND_B32       : DS_1A1D_NORET_mc<"ds_and_b32">;
448defm DS_OR_B32        : DS_1A1D_NORET_mc<"ds_or_b32">;
449defm DS_XOR_B32       : DS_1A1D_NORET_mc<"ds_xor_b32">;
450
451let SubtargetPredicate = HasLDSFPAtomicAdd in {
452defm DS_ADD_F32       : DS_1A1D_NORET_mc<"ds_add_f32">;
453}
454
455defm DS_MIN_F32       : DS_1A1D_NORET_mc<"ds_min_f32">;
456defm DS_MAX_F32       : DS_1A1D_NORET_mc<"ds_max_f32">;
457
458let mayLoad = 0 in {
459defm DS_WRITE_B8      : DS_1A1D_NORET_mc<"ds_write_b8">;
460defm DS_WRITE_B16     : DS_1A1D_NORET_mc<"ds_write_b16">;
461defm DS_WRITE_B32     : DS_1A1D_NORET_mc<"ds_write_b32">;
462defm DS_WRITE2_B32    : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">;
463defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">;
464
465
466let has_m0_read = 0 in {
467
468let SubtargetPredicate = HasD16LoadStore in {
469def DS_WRITE_B8_D16_HI  : DS_1A1D_NORET<"ds_write_b8_d16_hi">;
470def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;
471}
472
473} // End has_m0_read = 0
474
475let SubtargetPredicate = HasDSAddTid in {
476def DS_WRITE_ADDTID_B32 : DS_0A1D_NORET<"ds_write_addtid_b32">;
477}
478
479} // End mayLoad = 0
480
481let SubtargetPredicate = isGFX90APlus in {
482  defm DS_ADD_F64     : DS_1A1D_NORET_mc_gfx9<"ds_add_f64", VReg_64>;
483  defm DS_ADD_RTN_F64 : DS_1A1D_RET_mc_gfx9<"ds_add_rtn_f64", VReg_64, "ds_add_f64">;
484} // End SubtargetPredicate = isGFX90APlus
485
486let SubtargetPredicate = HasAtomicDsPkAdd16Insts in {
487  defm DS_PK_ADD_F16      : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_f16">;
488  defm DS_PK_ADD_RTN_F16  : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_f16", VGPR_32, "ds_pk_add_f16">;
489  defm DS_PK_ADD_BF16     : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_bf16">;
490  defm DS_PK_ADD_RTN_BF16 : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_bf16", VGPR_32, "ds_pk_add_bf16">;
491} // End SubtargetPredicate = HasAtomicDsPkAdd16Insts
492
493defm DS_CMPSTORE_B32     : DS_1A2D_NORET_mc<"ds_cmpstore_b32">;
494defm DS_CMPSTORE_F32     : DS_1A2D_NORET_mc<"ds_cmpstore_f32">;
495defm DS_CMPSTORE_B64     : DS_1A2D_NORET_mc<"ds_cmpstore_b64", VReg_64>;
496defm DS_CMPSTORE_F64     : DS_1A2D_NORET_mc<"ds_cmpstore_f64", VReg_64>;
497defm DS_CMPSTORE_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b32", VGPR_32, "ds_cmpstore_b32">;
498defm DS_CMPSTORE_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f32", VGPR_32, "ds_cmpstore_f32">;
499defm DS_CMPSTORE_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b64", VReg_64, "ds_cmpstore_b64">;
500defm DS_CMPSTORE_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f64", VReg_64, "ds_cmpstore_f64">;
501
502defm DS_MSKOR_B32     : DS_1A2D_NORET_mc<"ds_mskor_b32">;
503defm DS_CMPST_B32     : DS_1A2D_NORET_mc<"ds_cmpst_b32">;
504defm DS_CMPST_F32     : DS_1A2D_NORET_mc<"ds_cmpst_f32">;
505
506defm DS_ADD_U64       : DS_1A1D_NORET_mc<"ds_add_u64", VReg_64>;
507defm DS_SUB_U64       : DS_1A1D_NORET_mc<"ds_sub_u64", VReg_64>;
508defm DS_RSUB_U64      : DS_1A1D_NORET_mc<"ds_rsub_u64", VReg_64>;
509defm DS_INC_U64       : DS_1A1D_NORET_mc<"ds_inc_u64", VReg_64>;
510defm DS_DEC_U64       : DS_1A1D_NORET_mc<"ds_dec_u64", VReg_64>;
511defm DS_MIN_I64       : DS_1A1D_NORET_mc<"ds_min_i64", VReg_64>;
512defm DS_MAX_I64       : DS_1A1D_NORET_mc<"ds_max_i64", VReg_64>;
513defm DS_MIN_U64       : DS_1A1D_NORET_mc<"ds_min_u64", VReg_64>;
514defm DS_MAX_U64       : DS_1A1D_NORET_mc<"ds_max_u64", VReg_64>;
515defm DS_AND_B64       : DS_1A1D_NORET_mc<"ds_and_b64", VReg_64>;
516defm DS_OR_B64        : DS_1A1D_NORET_mc<"ds_or_b64", VReg_64>;
517defm DS_XOR_B64       : DS_1A1D_NORET_mc<"ds_xor_b64", VReg_64>;
518defm DS_MSKOR_B64     : DS_1A2D_NORET_mc<"ds_mskor_b64", VReg_64>;
519let mayLoad = 0 in {
520defm DS_WRITE_B64     : DS_1A1D_NORET_mc<"ds_write_b64", VReg_64>;
521defm DS_WRITE2_B64    : DS_1A2D_Off8_NORET_mc<"ds_write2_b64", VReg_64>;
522defm DS_WRITE2ST64_B64: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b64", VReg_64>;
523}
524defm DS_CMPST_B64     : DS_1A2D_NORET_mc<"ds_cmpst_b64", VReg_64>;
525defm DS_CMPST_F64     : DS_1A2D_NORET_mc<"ds_cmpst_f64", VReg_64>;
526defm DS_MIN_F64       : DS_1A1D_NORET_mc<"ds_min_f64", VReg_64>;
527defm DS_MAX_F64       : DS_1A1D_NORET_mc<"ds_max_f64", VReg_64>;
528
529defm DS_ADD_RTN_U32   : DS_1A1D_RET_mc<"ds_add_rtn_u32", VGPR_32, "ds_add_u32">;
530
531let SubtargetPredicate = HasLDSFPAtomicAdd in {
532defm DS_ADD_RTN_F32   : DS_1A1D_RET_mc<"ds_add_rtn_f32", VGPR_32, "ds_add_f32">;
533}
534defm DS_SUB_RTN_U32   : DS_1A1D_RET_mc<"ds_sub_rtn_u32", VGPR_32, "ds_sub_u32">;
535defm DS_RSUB_RTN_U32  : DS_1A1D_RET_mc<"ds_rsub_rtn_u32", VGPR_32, "ds_rsub_u32">;
536defm DS_INC_RTN_U32   : DS_1A1D_RET_mc<"ds_inc_rtn_u32", VGPR_32, "ds_inc_u32">;
537defm DS_DEC_RTN_U32   : DS_1A1D_RET_mc<"ds_dec_rtn_u32", VGPR_32, "ds_dec_u32">;
538defm DS_MIN_RTN_I32   : DS_1A1D_RET_mc<"ds_min_rtn_i32", VGPR_32, "ds_min_i32">;
539defm DS_MAX_RTN_I32   : DS_1A1D_RET_mc<"ds_max_rtn_i32", VGPR_32, "ds_max_i32">;
540defm DS_MIN_RTN_U32   : DS_1A1D_RET_mc<"ds_min_rtn_u32", VGPR_32, "ds_min_u32">;
541defm DS_MAX_RTN_U32   : DS_1A1D_RET_mc<"ds_max_rtn_u32", VGPR_32, "ds_max_u32">;
542defm DS_AND_RTN_B32   : DS_1A1D_RET_mc<"ds_and_rtn_b32", VGPR_32, "ds_and_b32">;
543defm DS_OR_RTN_B32    : DS_1A1D_RET_mc<"ds_or_rtn_b32", VGPR_32, "ds_or_b32">;
544defm DS_XOR_RTN_B32   : DS_1A1D_RET_mc<"ds_xor_rtn_b32", VGPR_32, "ds_xor_b32">;
545defm DS_MSKOR_RTN_B32 : DS_1A2D_RET_mc<"ds_mskor_rtn_b32", VGPR_32, "ds_mskor_b32">;
546defm DS_CMPST_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_b32", VGPR_32, "ds_cmpst_b32">;
547defm DS_CMPST_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_f32", VGPR_32, "ds_cmpst_f32">;
548defm DS_MIN_RTN_F32   : DS_1A1D_RET_mc<"ds_min_rtn_f32", VGPR_32, "ds_min_f32">;
549defm DS_MAX_RTN_F32   : DS_1A1D_RET_mc<"ds_max_rtn_f32", VGPR_32, "ds_max_f32">;
550
551defm DS_WRXCHG_RTN_B32 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b32">;
552defm DS_WRXCHG2_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b32", VReg_64, VGPR_32>;
553defm DS_WRXCHG2ST64_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b32", VReg_64, VGPR_32>;
554
555defm DS_ADD_RTN_U64  : DS_1A1D_RET_mc<"ds_add_rtn_u64", VReg_64, "ds_add_u64">;
556defm DS_SUB_RTN_U64  : DS_1A1D_RET_mc<"ds_sub_rtn_u64", VReg_64, "ds_sub_u64">;
557defm DS_RSUB_RTN_U64  : DS_1A1D_RET_mc<"ds_rsub_rtn_u64", VReg_64, "ds_rsub_u64">;
558defm DS_INC_RTN_U64   : DS_1A1D_RET_mc<"ds_inc_rtn_u64", VReg_64, "ds_inc_u64">;
559defm DS_DEC_RTN_U64   : DS_1A1D_RET_mc<"ds_dec_rtn_u64", VReg_64, "ds_dec_u64">;
560defm DS_MIN_RTN_I64    : DS_1A1D_RET_mc<"ds_min_rtn_i64", VReg_64, "ds_min_i64">;
561defm DS_MAX_RTN_I64    : DS_1A1D_RET_mc<"ds_max_rtn_i64", VReg_64, "ds_max_i64">;
562defm DS_MIN_RTN_U64   : DS_1A1D_RET_mc<"ds_min_rtn_u64", VReg_64, "ds_min_u64">;
563defm DS_MAX_RTN_U64   : DS_1A1D_RET_mc<"ds_max_rtn_u64", VReg_64, "ds_max_u64">;
564defm DS_AND_RTN_B64    : DS_1A1D_RET_mc<"ds_and_rtn_b64", VReg_64, "ds_and_b64">;
565defm DS_OR_RTN_B64     : DS_1A1D_RET_mc<"ds_or_rtn_b64", VReg_64, "ds_or_b64">;
566defm DS_XOR_RTN_B64    : DS_1A1D_RET_mc<"ds_xor_rtn_b64", VReg_64, "ds_xor_b64">;
567defm DS_MSKOR_RTN_B64  : DS_1A2D_RET_mc<"ds_mskor_rtn_b64", VReg_64, "ds_mskor_b64">;
568defm DS_CMPST_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_b64", VReg_64, "ds_cmpst_b64">;
569defm DS_CMPST_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_f64", VReg_64, "ds_cmpst_f64">;
570defm DS_MIN_RTN_F64    : DS_1A1D_RET_mc<"ds_min_rtn_f64", VReg_64, "ds_min_f64">;
571defm DS_MAX_RTN_F64    : DS_1A1D_RET_mc<"ds_max_rtn_f64", VReg_64, "ds_max_f64">;
572
573defm DS_WRXCHG_RTN_B64 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b64", VReg_64>;
574defm DS_WRXCHG2_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b64", VReg_128, VReg_64>;
575defm DS_WRXCHG2ST64_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b64", VReg_128, VReg_64>;
576
577let isConvergent = 1, usesCustomInserter = 1 in {
578def DS_GWS_INIT       : DS_GWS_1D<"ds_gws_init"> {
579  let mayLoad = 0;
580}
581def DS_GWS_SEMA_V     : DS_GWS_0D<"ds_gws_sema_v">;
582def DS_GWS_SEMA_BR    : DS_GWS_1D<"ds_gws_sema_br">;
583def DS_GWS_SEMA_P     : DS_GWS_0D<"ds_gws_sema_p">;
584def DS_GWS_BARRIER    : DS_GWS_1D<"ds_gws_barrier">;
585}
586
587let SubtargetPredicate = HasDsSrc2Insts in {
588def DS_ADD_SRC2_U32   : DS_1A<"ds_add_src2_u32">;
589def DS_SUB_SRC2_U32   : DS_1A<"ds_sub_src2_u32">;
590def DS_RSUB_SRC2_U32  : DS_1A<"ds_rsub_src2_u32">;
591def DS_INC_SRC2_U32   : DS_1A<"ds_inc_src2_u32">;
592def DS_DEC_SRC2_U32   : DS_1A<"ds_dec_src2_u32">;
593def DS_MIN_SRC2_I32   : DS_1A<"ds_min_src2_i32">;
594def DS_MAX_SRC2_I32   : DS_1A<"ds_max_src2_i32">;
595def DS_MIN_SRC2_U32   : DS_1A<"ds_min_src2_u32">;
596def DS_MAX_SRC2_U32   : DS_1A<"ds_max_src2_u32">;
597def DS_AND_SRC2_B32   : DS_1A<"ds_and_src2_b32">;
598def DS_OR_SRC2_B32    : DS_1A<"ds_or_src2_b32">;
599def DS_XOR_SRC2_B32   : DS_1A<"ds_xor_src2_b32">;
600def DS_MIN_SRC2_F32   : DS_1A<"ds_min_src2_f32">;
601def DS_MAX_SRC2_F32   : DS_1A<"ds_max_src2_f32">;
602
603def DS_ADD_SRC2_U64   : DS_1A<"ds_add_src2_u64">;
604def DS_SUB_SRC2_U64   : DS_1A<"ds_sub_src2_u64">;
605def DS_RSUB_SRC2_U64  : DS_1A<"ds_rsub_src2_u64">;
606def DS_INC_SRC2_U64   : DS_1A<"ds_inc_src2_u64">;
607def DS_DEC_SRC2_U64   : DS_1A<"ds_dec_src2_u64">;
608def DS_MIN_SRC2_I64   : DS_1A<"ds_min_src2_i64">;
609def DS_MAX_SRC2_I64   : DS_1A<"ds_max_src2_i64">;
610def DS_MIN_SRC2_U64   : DS_1A<"ds_min_src2_u64">;
611def DS_MAX_SRC2_U64   : DS_1A<"ds_max_src2_u64">;
612def DS_AND_SRC2_B64   : DS_1A<"ds_and_src2_b64">;
613def DS_OR_SRC2_B64    : DS_1A<"ds_or_src2_b64">;
614def DS_XOR_SRC2_B64   : DS_1A<"ds_xor_src2_b64">;
615def DS_MIN_SRC2_F64   : DS_1A<"ds_min_src2_f64">;
616def DS_MAX_SRC2_F64   : DS_1A<"ds_max_src2_f64">;
617
618def DS_WRITE_SRC2_B32 : DS_1A<"ds_write_src2_b32">;
619def DS_WRITE_SRC2_B64 : DS_1A<"ds_write_src2_b64">;
620} // End SubtargetPredicate = HasDsSrc2Insts
621
622let Uses = [EXEC], mayLoad = 0, mayStore = 0, isConvergent = 1 in {
623def DS_SWIZZLE_B32 : DS_1A_RET <"ds_swizzle_b32", VGPR_32, 0, Swizzle>;
624}
625
626let mayStore = 0 in {
627defm DS_READ_I8      : DS_1A_RET_mc<"ds_read_i8">;
628defm DS_READ_U8      : DS_1A_RET_mc<"ds_read_u8">;
629defm DS_READ_I16     : DS_1A_RET_mc<"ds_read_i16">;
630defm DS_READ_U16     : DS_1A_RET_mc<"ds_read_u16">;
631defm DS_READ_B32     : DS_1A_RET_mc<"ds_read_b32">;
632defm DS_READ_B64     : DS_1A_RET_mc<"ds_read_b64", VReg_64>;
633
634defm DS_READ2_B32    : DS_1A_Off8_RET_mc<"ds_read2_b32", VReg_64>;
635defm DS_READ2ST64_B32: DS_1A_Off8_RET_mc<"ds_read2st64_b32", VReg_64>;
636
637defm DS_READ2_B64    : DS_1A_Off8_RET_mc<"ds_read2_b64", VReg_128>;
638defm DS_READ2ST64_B64: DS_1A_Off8_RET_mc<"ds_read2st64_b64", VReg_128>;
639
640let has_m0_read = 0 in {
641let SubtargetPredicate = HasD16LoadStore, TiedSourceNotRead = 1 in {
642def DS_READ_U8_D16     : DS_1A_RET_Tied<"ds_read_u8_d16">;
643def DS_READ_U8_D16_HI  : DS_1A_RET_Tied<"ds_read_u8_d16_hi">;
644def DS_READ_I8_D16     : DS_1A_RET_Tied<"ds_read_i8_d16">;
645def DS_READ_I8_D16_HI  : DS_1A_RET_Tied<"ds_read_i8_d16_hi">;
646def DS_READ_U16_D16    : DS_1A_RET_Tied<"ds_read_u16_d16">;
647def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">;
648}
649} // End has_m0_read = 0
650
651let SubtargetPredicate = HasDSAddTid in {
652def DS_READ_ADDTID_B32 : DS_0A_RET<"ds_read_addtid_b32">;
653}
654
655} // End mayStore = 0
656
657def DS_CONSUME       : DS_0A_RET<"ds_consume">;
658def DS_APPEND        : DS_0A_RET<"ds_append">;
659
660let SubtargetPredicate = isNotGFX90APlus in
661def DS_ORDERED_COUNT : DS_1A_RET_GDS<"ds_ordered_count">;
662
663//===----------------------------------------------------------------------===//
664// Instruction definitions for CI and newer.
665//===----------------------------------------------------------------------===//
666
667let SubtargetPredicate = isGFX7Plus in {
668
669defm DS_WRAP_RTN_B32 : DS_1A2D_RET_mc<"ds_wrap_rtn_b32", VGPR_32>;
670defm DS_CONDXCHG32_RTN_B64 : DS_1A1D_RET_mc<"ds_condxchg32_rtn_b64", VReg_64>;
671
672let isConvergent = 1, usesCustomInserter = 1 in {
673def DS_GWS_SEMA_RELEASE_ALL : DS_GWS_0D<"ds_gws_sema_release_all">;
674}
675
676let mayStore = 0 in {
677defm DS_READ_B96 : DS_1A_RET_mc<"ds_read_b96", VReg_96>;
678defm DS_READ_B128: DS_1A_RET_mc<"ds_read_b128", VReg_128>;
679} // End mayStore = 0
680
681let mayLoad = 0 in {
682defm DS_WRITE_B96 : DS_1A1D_NORET_mc<"ds_write_b96", VReg_96>;
683defm DS_WRITE_B128 : DS_1A1D_NORET_mc<"ds_write_b128", VReg_128>;
684} // End mayLoad = 0
685
686def DS_NOP : DS_VOID<"ds_nop">;
687
688} // let SubtargetPredicate = isGFX7Plus
689
690//===----------------------------------------------------------------------===//
691// Instruction definitions for VI and newer.
692//===----------------------------------------------------------------------===//
693
694let SubtargetPredicate = isGFX8Plus in {
695
696let Uses = [EXEC] in {
697def DS_PERMUTE_B32  : DS_1A1D_PERMUTE <"ds_permute_b32",
698                                       int_amdgcn_ds_permute>;
699def DS_BPERMUTE_B32 : DS_1A1D_PERMUTE <"ds_bpermute_b32",
700                                       int_amdgcn_ds_bpermute>;
701}
702
703} // let SubtargetPredicate = isGFX8Plus
704
705let SubtargetPredicate = HasLDSFPAtomicAdd, OtherPredicates = [HasDsSrc2Insts] in {
706def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;
707}
708
709
710//===----------------------------------------------------------------------===//
711// Instruction definitions for GFX11 and newer.
712//===----------------------------------------------------------------------===//
713
714let SubtargetPredicate = isGFX11Plus in {
715
716def DS_ADD_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_add_gs_reg_rtn", VReg_64, VGPR_32>;
717def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VReg_64, VGPR_32>;
718def DS_BVH_STACK_RTN_B32 : DS_BVH_STACK<"ds_bvh_stack_rtn_b32">;
719
720} // let SubtargetPredicate = isGFX11Plus
721
722//===----------------------------------------------------------------------===//
723// DS Patterns
724//===----------------------------------------------------------------------===//
725
726def : GCNPat <
727  (int_amdgcn_ds_swizzle i32:$src, timm:$offset16),
728  (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0))
729>;
730
731class DSReadPat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
732  (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset))),
733  (inst $ptr, offset:$offset, (i1 gds))
734>;
735
736multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
737
738  let OtherPredicates = [LDSRequiresM0Init] in {
739    def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
740  }
741
742  let OtherPredicates = [NotLDSRequiresM0Init] in {
743    def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
744  }
745}
746
747class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <
748  (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$in),
749  (inst $ptr, offset:$offset, (i1 0), $in)
750>;
751
752defm : DSReadPat_mc <DS_READ_I8, i32, "sextloadi8_local">;
753defm : DSReadPat_mc <DS_READ_I8,  i16, "sextloadi8_local">;
754defm : DSReadPat_mc <DS_READ_U8,  i32, "extloadi8_local">;
755defm : DSReadPat_mc <DS_READ_U8,  i32, "zextloadi8_local">;
756defm : DSReadPat_mc <DS_READ_U8,  i16, "extloadi8_local">;
757defm : DSReadPat_mc <DS_READ_U8,  i16, "zextloadi8_local">;
758defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
759defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
760defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
761defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
762defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
763
764foreach vt = Reg32Types.types in {
765defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
766}
767
768defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_8_local">;
769defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_8_local">;
770defm : DSReadPat_mc <DS_READ_U16, i16, "atomic_load_16_local">;
771defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_16_local">;
772defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
773defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
774
775let OtherPredicates = [D16PreservesUnusedBits] in {
776def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
777def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
778def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
779def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;
780def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2i16>;
781def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2f16>;
782
783def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2i16>;
784def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2f16>;
785def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2i16>;
786def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2f16>;
787def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2i16>;
788def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2f16>;
789}
790
791class DSWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
792  (frag vt:$value, (DS1Addr1Offset i32:$ptr, i32:$offset)),
793  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
794>;
795
796multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
797  let OtherPredicates = [LDSRequiresM0Init] in {
798    def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
799  }
800
801  let OtherPredicates = [NotLDSRequiresM0Init] in {
802    def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
803  }
804}
805
806// Irritatingly, atomic_store reverses the order of operands from a
807// normal store.
808class DSAtomicWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
809  (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),
810  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 0))
811>;
812
813multiclass DSAtomicWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
814  let OtherPredicates = [LDSRequiresM0Init] in {
815    def : DSAtomicWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
816  }
817
818  let OtherPredicates = [NotLDSRequiresM0Init] in {
819    def : DSAtomicWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
820  }
821}
822
823defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
824defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
825defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
826defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
827
828foreach vt = Reg32Types.types in {
829defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
830}
831
832defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
833defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
834defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
835defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
836defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
837defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
838
839let OtherPredicates = [HasD16LoadStore] in {
840def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
841def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;
842}
843
844class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
845  (vt:$value (frag (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
846  (inst $ptr, $offset0, $offset1, (i1 0))
847>;
848
849class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
850  (frag vt:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
851  (inst $ptr, (i32 (EXTRACT_SUBREG VReg_64:$value, sub0)),
852              (i32 (EXTRACT_SUBREG VReg_64:$value, sub1)), $offset0, $offset1,
853              (i1 0))
854>;
855
856class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
857  (vt:$value (frag (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
858  (inst $ptr, $offset0, $offset1, (i1 0))
859>;
860
861class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
862  (frag vt:$value, (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
863  (inst $ptr, (i64 (EXTRACT_SUBREG VReg_128:$value, sub0_sub1)),
864              (i64 (EXTRACT_SUBREG VReg_128:$value, sub2_sub3)), $offset0, $offset1,
865              (i1 0))
866>;
867
868multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {
869  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
870    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32, vt, load_local_m0>;
871    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_local_m0>;
872  }
873
874  let OtherPredicates = [NotLDSRequiresM0Init] in {
875    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, vt, load_local>;
876    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, vt, store_local>;
877  }
878}
879
880multiclass DS128Bit8ByteAlignedPat_mc<ValueType vt> {
881  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
882    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64, vt, load_local_m0>;
883    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_local_m0>;
884  }
885
886  let OtherPredicates = [NotLDSRequiresM0Init] in {
887    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64_gfx9, vt, load_local>;
888    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64_gfx9, vt, store_local>;
889  }
890}
891
892// v2i32 loads are split into i32 loads on SI during lowering, due to a bug
893// related to bounds checking.
894foreach vt = VReg_64.RegTypes in {
895defm : DS64Bit4ByteAlignedPat_mc<vt>;
896}
897
898foreach vt = VReg_128.RegTypes in {
899defm : DS128Bit8ByteAlignedPat_mc<vt>;
900}
901
902// Prefer ds_read over ds_read2 and ds_write over ds_write2, all other things
903// being equal, because it has a larger immediate offset range.
904let AddedComplexity = 100 in {
905
906foreach vt = VReg_64.RegTypes in {
907defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
908defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
909}
910
911let SubtargetPredicate = isGFX7Plus in {
912
913foreach vt = VReg_96.RegTypes in {
914defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
915defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
916}
917
918foreach vt = VReg_128.RegTypes in {
919defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
920defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
921}
922
923let SubtargetPredicate = HasUnalignedAccessMode in {
924
925// Select 64 bit loads and stores aligned less than 4 as a single ds_read_b64/
926// ds_write_b64 instruction as this is faster than ds_read2_b32/ds_write2_b32
927// which would be used otherwise. In this case a b32 access would still be
928// misaligned, but we will have 2 of them.
929foreach vt = VReg_64.RegTypes in {
930defm : DSReadPat_mc <DS_READ_B64, vt, "load_align_less_than_4_local">;
931defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align_less_than_4_local">;
932}
933
934// Selection will split most of the unaligned 3 dword accesses due to performance
935// reasons when beneficial. Keep these two patterns for the rest of the cases.
936foreach vt = VReg_96.RegTypes in {
937defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
938defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
939}
940
941// Select 128 bit loads and stores aligned less than 4 as a single ds_read_b128/
942// ds_write_b128 instruction as this is faster than ds_read2_b64/ds_write2_b64
943// which would be used otherwise. In this case a b64 access would still be
944// misaligned, but we will have 2 of them.
945foreach vt = VReg_128.RegTypes in {
946defm : DSReadPat_mc <DS_READ_B128, vt, "load_align_less_than_4_local">;
947defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align_less_than_4_local">;
948}
949
950} // End SubtargetPredicate = HasUnalignedAccessMode
951
952} // End SubtargetPredicate = isGFX7Plus
953
954} // End AddedComplexity = 100
955
956class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, int complexity = 0,
957  bit gds=0> : GCNPat <(frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),
958  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))> {
959  let AddedComplexity = complexity;
960}
961
962multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
963  let OtherPredicates = [LDSRequiresM0Init] in {
964    def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
965  }
966
967  let OtherPredicates = [NotLDSRequiresM0Init] in {
968    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
969                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
970  }
971
972  def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
973                       /* complexity */ 0, /* gds */ 1>;
974}
975
976multiclass DSAtomicRetNoRetPat_mc<DS_Pseudo inst, DS_Pseudo noRetInst,
977                                  ValueType vt, string frag> {
978  let OtherPredicates = [LDSRequiresM0Init] in {
979    def : DSAtomicRetPat<inst, vt,
980                         !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
981    def : DSAtomicRetPat<noRetInst, vt,
982                         !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size), /* complexity */ 1>;
983  }
984
985  let OtherPredicates = [NotLDSRequiresM0Init] in {
986    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
987                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
988    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
989                         !cast<PatFrag>(frag#"_local_noret_"#vt.Size), /* complexity */ 1>;
990  }
991
992  def : DSAtomicRetPat<inst, vt,
993                       !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
994                       /* complexity */ 0, /* gds */ 1>;
995  def : DSAtomicRetPat<noRetInst, vt,
996                       !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
997                       /* complexity */ 1, /* gds */ 1>;
998}
999
1000
1001
1002let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1003// Caution, the order of src and cmp is the *opposite* of the BUFFER_ATOMIC_CMPSWAP opcode.
1004class DSAtomicCmpXChgSwapped<DS_Pseudo inst, ValueType vt, PatFrag frag,
1005  int complexity = 0, bit gds=0> : GCNPat<
1006  (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap),
1007  (inst $ptr, getVregSrcForVT<vt>.ret:$cmp, getVregSrcForVT<vt>.ret:$swap, offset:$offset, (i1 gds))> {
1008  let AddedComplexity = complexity;
1009}
1010
1011multiclass DSAtomicCmpXChgSwapped_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt,
1012                                     string frag> {
1013  let OtherPredicates = [LDSRequiresM0Init] in {
1014    def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
1015    def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size),
1016                                 /* complexity */ 1>;
1017  }
1018
1019  let OtherPredicates = [NotLDSRequiresM0Init] in {
1020    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1021                                 !cast<PatFrag>(frag#"_local_"#vt.Size)>;
1022    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1023                                 !cast<PatFrag>(frag#"_local_noret_"#vt.Size),
1024                                 /* complexity */ 1>;
1025  }
1026
1027  def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1028                               /* complexity */ 0, /* gds */ 1>;
1029  def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1030                               /* complexity */ 1, /* gds */ 1>;
1031}
1032} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1033
1034let SubtargetPredicate = isGFX11Plus in {
1035// The order of src and cmp agrees with the BUFFER_ATOMIC_CMPSWAP opcode.
1036class DSAtomicCmpXChg<DS_Pseudo inst, ValueType vt, PatFrag frag,
1037  int complexity = 0, bit gds=0> : GCNPat<
1038  (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap),
1039  (inst $ptr, getVregSrcForVT<vt>.ret:$swap, getVregSrcForVT<vt>.ret:$cmp, offset:$offset, (i1 gds))> {
1040  let AddedComplexity = complexity;
1041}
1042
1043multiclass DSAtomicCmpXChg_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt, string frag> {
1044
1045  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1046                        !cast<PatFrag>(frag#"_local_"#vt.Size)>;
1047  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1048                        !cast<PatFrag>(frag#"_local_noret_"#vt.Size), /* complexity */ 1>;
1049
1050  def : DSAtomicCmpXChg<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1051                        /* complexity */ 0, /* gds */ 1>;
1052  def : DSAtomicCmpXChg<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1053                        /* complexity */ 1, /* gds */ 1>;
1054}
1055} // End SubtargetPredicate = isGFX11Plus
1056
1057// 32-bit atomics.
1058defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B32, i32, "atomic_swap">;
1059defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U32, DS_ADD_U32, i32, "atomic_load_add">;
1060defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U32, DS_SUB_U32, i32, "atomic_load_sub">;
1061defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_load_uinc_wrap">;
1062defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_load_udec_wrap">;
1063defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B32, DS_AND_B32, i32, "atomic_load_and">;
1064defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B32, DS_OR_B32, i32, "atomic_load_or">;
1065defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B32, DS_XOR_B32, i32, "atomic_load_xor">;
1066defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I32, DS_MIN_I32, i32, "atomic_load_min">;
1067defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I32, DS_MAX_I32, i32, "atomic_load_max">;
1068defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U32, DS_MIN_U32, i32, "atomic_load_umin">;
1069defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U32, DS_MAX_U32, i32, "atomic_load_umax">;
1070defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F32, DS_MIN_F32, f32, "atomic_load_fmin">;
1071defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F32, DS_MAX_F32, f32, "atomic_load_fmax">;
1072
1073let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1074defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B32, DS_CMPST_B32, i32, "atomic_cmp_swap">;
1075}
1076
1077let SubtargetPredicate = isGFX11Plus in {
1078defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B32, DS_CMPSTORE_B32, i32, "atomic_cmp_swap">;
1079}
1080
1081let SubtargetPredicate = HasLDSFPAtomicAdd in {
1082defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_F32, DS_ADD_F32, f32, "atomic_load_fadd">;
1083}
1084
1085// 64-bit atomics.
1086defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B64, i64, "atomic_swap">;
1087defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U64, DS_ADD_U64, i64, "atomic_load_add">;
1088defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U64, DS_SUB_U64, i64, "atomic_load_sub">;
1089defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_load_uinc_wrap">;
1090defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_load_udec_wrap">;
1091defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B64, DS_AND_B64, i64, "atomic_load_and">;
1092defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B64, DS_OR_B64, i64, "atomic_load_or">;
1093defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B64, DS_XOR_B64, i64, "atomic_load_xor">;
1094defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I64, DS_MIN_I64, i64, "atomic_load_min">;
1095defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I64, DS_MAX_I64, i64, "atomic_load_max">;
1096defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U64, DS_MIN_U64, i64, "atomic_load_umin">;
1097defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U64, DS_MAX_U64, i64, "atomic_load_umax">;
1098defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F64, DS_MIN_F64, f64, "atomic_load_fmin">;
1099defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F64, DS_MAX_F64, f64, "atomic_load_fmax">;
1100
1101let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1102defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B64, DS_CMPST_B64, i64, "atomic_cmp_swap">;
1103} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1104
1105let SubtargetPredicate = isGFX11Plus in {
1106defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B64, DS_CMPSTORE_B64, i64, "atomic_cmp_swap">;
1107} // End SubtargetPredicate = isGFX11Plus
1108
1109let SubtargetPredicate = isGFX90APlus in {
1110def : DSAtomicRetPat<DS_ADD_RTN_F64, f64, atomic_load_fadd_local_64>;
1111let AddedComplexity = 1 in
1112def : DSAtomicRetPat<DS_ADD_F64, f64, atomic_load_fadd_local_noret_64>;
1113
1114class DSAtomicRetPatIntrinsic<DS_Pseudo inst, ValueType vt, PatFrag frag,
1115  bit gds=0> : GCNPat <
1116  (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value)),
1117  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))> {
1118}
1119
1120def : DSAtomicRetPatIntrinsic<DS_ADD_RTN_F64, f64, int_amdgcn_flat_atomic_fadd_local_addrspace>;
1121let AddedComplexity = 1 in
1122def : DSAtomicRetPatIntrinsic<DS_ADD_F64, f64, int_amdgcn_flat_atomic_fadd_noret_local_addrspace>;
1123}
1124
1125let SubtargetPredicate = HasAtomicDsPkAdd16Insts in {
1126def : DSAtomicRetPat<DS_PK_ADD_RTN_F16, v2f16, atomic_load_fadd_v2f16_local_32>;
1127let AddedComplexity = 1 in
1128def : DSAtomicRetPat<DS_PK_ADD_F16, v2f16, atomic_load_fadd_v2f16_local_noret_32>;
1129def : GCNPat <
1130  (v2i16 (int_amdgcn_ds_fadd_v2bf16 i32:$ptr, v2i16:$src)),
1131  (DS_PK_ADD_RTN_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1132>;
1133let AddedComplexity = 1 in
1134def : GCNPat <
1135  (v2i16 (int_amdgcn_ds_fadd_v2bf16_noret i32:$ptr, v2i16:$src)),
1136  (DS_PK_ADD_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1137>;
1138} // End SubtargetPredicate = HasAtomicDsPkAdd16Insts
1139
1140def : Pat <
1141  (SIds_ordered_count i32:$value, i16:$offset),
1142  (DS_ORDERED_COUNT $value, (as_i16imm $offset))
1143>;
1144
1145def : GCNPat <
1146  (i64 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),
1147  (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))
1148>;
1149
1150def : GCNPat <
1151  (i32 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),
1152  (EXTRACT_SUBREG
1153    (i64 (COPY_TO_REGCLASS
1154      (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),
1155      VReg_64)),
1156    sub0)
1157>;
1158
1159def : GCNPat <
1160  (i64 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),
1161  (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))
1162>;
1163
1164def : GCNPat <
1165  (i32 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),
1166  (EXTRACT_SUBREG
1167    (i64 (COPY_TO_REGCLASS
1168      (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),
1169      VReg_64)),
1170    sub0)
1171>;
1172
1173//===----------------------------------------------------------------------===//
1174// Target-specific instruction encodings.
1175//===----------------------------------------------------------------------===//
1176
1177//===----------------------------------------------------------------------===//
1178// Base ENC_DS for GFX6, GFX7, GFX10, GFX11.
1179//===----------------------------------------------------------------------===//
1180
1181class Base_DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op, DS_Pseudo ps, int ef, string opName = ps.Mnemonic> :
1182    DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> {
1183
1184  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1185  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1186  let Inst{17}    = !if(ps.has_gds, gds, ps.gdsValue);
1187  let Inst{25-18} = op;
1188  let Inst{31-26} = 0x36;
1189  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1190  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1191  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1192  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1193}
1194
1195//===----------------------------------------------------------------------===//
1196// GFX11.
1197//===----------------------------------------------------------------------===//
1198
1199let AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11" in {
1200  multiclass DS_Real_gfx11<bits<8> op>  {
1201    def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1202                                              SIEncodingFamily.GFX11>;
1203  }
1204
1205  multiclass DS_Real_Renamed_gfx11<bits<8> op, DS_Pseudo backing_pseudo, string real_name> {
1206     def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, backing_pseudo, SIEncodingFamily.GFX11, real_name>,
1207               MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX11Plus]>;
1208  }
1209} // End AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11"
1210
1211defm DS_STORE_B32                        : DS_Real_Renamed_gfx11<0x00d, DS_WRITE_B32, "ds_store_b32">;
1212defm DS_STORE_2ADDR_B32                  : DS_Real_Renamed_gfx11<0x00e, DS_WRITE2_B32, "ds_store_2addr_b32">;
1213defm DS_STORE_2ADDR_STRIDE64_B32         : DS_Real_Renamed_gfx11<0x00f, DS_WRITE2ST64_B32, "ds_store_2addr_stride64_b32">;
1214defm DS_STORE_B8                         : DS_Real_Renamed_gfx11<0x01e, DS_WRITE_B8, "ds_store_b8">;
1215defm DS_STORE_B16                        : DS_Real_Renamed_gfx11<0x01f, DS_WRITE_B16, "ds_store_b16">;
1216defm DS_STOREXCHG_RTN_B32                : DS_Real_Renamed_gfx11<0x02d, DS_WRXCHG_RTN_B32, "ds_storexchg_rtn_b32">;
1217defm DS_STOREXCHG_2ADDR_RTN_B32          : DS_Real_Renamed_gfx11<0x02e, DS_WRXCHG2_RTN_B32, "ds_storexchg_2addr_rtn_b32">;
1218defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32 : DS_Real_Renamed_gfx11<0x02f, DS_WRXCHG2ST64_RTN_B32, "ds_storexchg_2addr_stride64_rtn_b32">;
1219defm DS_LOAD_B32                         : DS_Real_Renamed_gfx11<0x036, DS_READ_B32, "ds_load_b32">;
1220defm DS_LOAD_2ADDR_B32                   : DS_Real_Renamed_gfx11<0x037, DS_READ2_B32, "ds_load_2addr_b32">;
1221defm DS_LOAD_2ADDR_STRIDE64_B32          : DS_Real_Renamed_gfx11<0x038, DS_READ2ST64_B32, "ds_load_2addr_stride64_b32">;
1222defm DS_LOAD_I8                          : DS_Real_Renamed_gfx11<0x039, DS_READ_I8, "ds_load_i8">;
1223defm DS_LOAD_U8                          : DS_Real_Renamed_gfx11<0x03a, DS_READ_U8, "ds_load_u8">;
1224defm DS_LOAD_I16                         : DS_Real_Renamed_gfx11<0x03b, DS_READ_I16, "ds_load_i16">;
1225defm DS_LOAD_U16                         : DS_Real_Renamed_gfx11<0x03c, DS_READ_U16, "ds_load_u16">;
1226defm DS_STORE_B64                        : DS_Real_Renamed_gfx11<0x04d, DS_WRITE_B64, "ds_store_b64">;
1227defm DS_STORE_2ADDR_B64                  : DS_Real_Renamed_gfx11<0x04e, DS_WRITE2_B64, "ds_store_2addr_b64">;
1228defm DS_STORE_2ADDR_STRIDE64_B64         : DS_Real_Renamed_gfx11<0x04f, DS_WRITE2ST64_B64, "ds_store_2addr_stride64_b64">;
1229defm DS_STOREXCHG_RTN_B64                : DS_Real_Renamed_gfx11<0x06d, DS_WRXCHG_RTN_B64, "ds_storexchg_rtn_b64">;
1230defm DS_STOREXCHG_2ADDR_RTN_B64          : DS_Real_Renamed_gfx11<0x06e, DS_WRXCHG2_RTN_B64, "ds_storexchg_2addr_rtn_b64">;
1231defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64 : DS_Real_Renamed_gfx11<0x06f, DS_WRXCHG2ST64_RTN_B64, "ds_storexchg_2addr_stride64_rtn_b64">;
1232defm DS_LOAD_B64                         : DS_Real_Renamed_gfx11<0x076, DS_READ_B64, "ds_load_b64">;
1233defm DS_LOAD_2ADDR_B64                   : DS_Real_Renamed_gfx11<0x077, DS_READ2_B64, "ds_load_2addr_b64">;
1234defm DS_LOAD_2ADDR_STRIDE64_B64          : DS_Real_Renamed_gfx11<0x078, DS_READ2ST64_B64, "ds_load_2addr_stride64_b64">;
1235defm DS_STORE_B8_D16_HI                  : DS_Real_Renamed_gfx11<0x0a0, DS_WRITE_B8_D16_HI, "ds_store_b8_d16_hi">;
1236defm DS_STORE_B16_D16_HI                 : DS_Real_Renamed_gfx11<0x0a1, DS_WRITE_B16_D16_HI, "ds_store_b16_d16_hi">;
1237defm DS_LOAD_U8_D16                      : DS_Real_Renamed_gfx11<0x0a2, DS_READ_U8_D16, "ds_load_u8_d16">;
1238defm DS_LOAD_U8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a3, DS_READ_U8_D16_HI, "ds_load_u8_d16_hi">;
1239defm DS_LOAD_I8_D16                      : DS_Real_Renamed_gfx11<0x0a4, DS_READ_I8_D16, "ds_load_i8_d16">;
1240defm DS_LOAD_I8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a5, DS_READ_I8_D16_HI, "ds_load_i8_d16_hi">;
1241defm DS_LOAD_U16_D16                     : DS_Real_Renamed_gfx11<0x0a6, DS_READ_U16_D16, "ds_load_u16_d16">;
1242defm DS_LOAD_U16_D16_HI                  : DS_Real_Renamed_gfx11<0x0a7, DS_READ_U16_D16_HI, "ds_load_u16_d16_hi">;
1243defm DS_STORE_ADDTID_B32                 : DS_Real_Renamed_gfx11<0x0b0, DS_WRITE_ADDTID_B32, "ds_store_addtid_b32">;
1244defm DS_LOAD_ADDTID_B32                  : DS_Real_Renamed_gfx11<0x0b1, DS_READ_ADDTID_B32, "ds_load_addtid_b32">;
1245defm DS_STORE_B96                        : DS_Real_Renamed_gfx11<0x0de, DS_WRITE_B96, "ds_store_b96">;
1246defm DS_STORE_B128                       : DS_Real_Renamed_gfx11<0x0df, DS_WRITE_B128, "ds_store_b128">;
1247defm DS_LOAD_B96                         : DS_Real_Renamed_gfx11<0x0fe, DS_READ_B96, "ds_load_b96">;
1248defm DS_LOAD_B128                        : DS_Real_Renamed_gfx11<0x0ff, DS_READ_B128, "ds_load_b128">;
1249
1250// DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped
1251// comparing to pre-GFX11.
1252// Note: the mnemonic alias is not generated to avoid a potential ambiguity due to the semantics change.
1253
1254defm DS_CMPSTORE_B32                     : DS_Real_gfx11<0x010>;
1255defm DS_CMPSTORE_F32                     : DS_Real_gfx11<0x011>;
1256defm DS_CMPSTORE_RTN_B32                 : DS_Real_gfx11<0x030>;
1257defm DS_CMPSTORE_RTN_F32                 : DS_Real_gfx11<0x031>;
1258defm DS_CMPSTORE_B64                     : DS_Real_gfx11<0x050>;
1259defm DS_CMPSTORE_F64                     : DS_Real_gfx11<0x051>;
1260defm DS_CMPSTORE_RTN_B64                 : DS_Real_gfx11<0x070>;
1261defm DS_CMPSTORE_RTN_F64                 : DS_Real_gfx11<0x071>;
1262
1263defm DS_ADD_RTN_F32                      : DS_Real_gfx11<0x079>;
1264defm DS_ADD_GS_REG_RTN                   : DS_Real_gfx11<0x07a>;
1265defm DS_SUB_GS_REG_RTN                   : DS_Real_gfx11<0x07b>;
1266defm DS_BVH_STACK_RTN_B32                : DS_Real_gfx11<0x0ad>;
1267
1268//===----------------------------------------------------------------------===//
1269// GFX10.
1270//===----------------------------------------------------------------------===//
1271
1272let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
1273  multiclass DS_Real_gfx10<bits<8> op>  {
1274    def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1275                                              SIEncodingFamily.GFX10>;
1276  }
1277} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
1278
1279defm DS_ADD_RTN_F32      : DS_Real_gfx10<0x055>;
1280defm DS_WRITE_B8_D16_HI  : DS_Real_gfx10<0x0a0>;
1281defm DS_WRITE_B16_D16_HI : DS_Real_gfx10<0x0a1>;
1282defm DS_READ_U8_D16      : DS_Real_gfx10<0x0a2>;
1283defm DS_READ_U8_D16_HI   : DS_Real_gfx10<0x0a3>;
1284defm DS_READ_I8_D16      : DS_Real_gfx10<0x0a4>;
1285defm DS_READ_I8_D16_HI   : DS_Real_gfx10<0x0a5>;
1286defm DS_READ_U16_D16     : DS_Real_gfx10<0x0a6>;
1287defm DS_READ_U16_D16_HI  : DS_Real_gfx10<0x0a7>;
1288defm DS_WRITE_ADDTID_B32 : DS_Real_gfx10<0x0b0>;
1289defm DS_READ_ADDTID_B32  : DS_Real_gfx10<0x0b1>;
1290
1291//===----------------------------------------------------------------------===//
1292// GFX10, GFX11.
1293//===----------------------------------------------------------------------===//
1294
1295multiclass DS_Real_gfx10_gfx11<bits<8> op> :
1296  DS_Real_gfx10<op>, DS_Real_gfx11<op>;
1297
1298defm DS_ADD_F32          : DS_Real_gfx10_gfx11<0x015>;
1299defm DS_ADD_SRC2_F32     : DS_Real_gfx10<0x095>;
1300defm DS_PERMUTE_B32      : DS_Real_gfx10_gfx11<0x0b2>;
1301defm DS_BPERMUTE_B32     : DS_Real_gfx10_gfx11<0x0b3>;
1302
1303//===----------------------------------------------------------------------===//
1304// GFX7, GFX10, GFX11.
1305//===----------------------------------------------------------------------===//
1306
1307let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1308  multiclass DS_Real_gfx7<bits<8> op> {
1309    def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1310                                             SIEncodingFamily.SI>;
1311  }
1312} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1313
1314multiclass DS_Real_gfx7_gfx10_gfx11<bits<8> op> :
1315  DS_Real_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1316
1317multiclass DS_Real_gfx7_gfx10<bits<8> op> :
1318  DS_Real_gfx7<op>, DS_Real_gfx10<op>;
1319
1320// FIXME-GFX7: Add tests when upstreaming this part.
1321defm DS_GWS_SEMA_RELEASE_ALL : DS_Real_gfx7_gfx10_gfx11<0x018>;
1322defm DS_WRAP_RTN_B32         : DS_Real_gfx7_gfx10_gfx11<0x034>;
1323defm DS_CONDXCHG32_RTN_B64   : DS_Real_gfx7_gfx10_gfx11<0x07e>;
1324defm DS_WRITE_B96            : DS_Real_gfx7_gfx10<0x0de>;
1325defm DS_WRITE_B128           : DS_Real_gfx7_gfx10<0x0df>;
1326defm DS_READ_B96             : DS_Real_gfx7_gfx10<0x0fe>;
1327defm DS_READ_B128            : DS_Real_gfx7_gfx10<0x0ff>;
1328
1329//===----------------------------------------------------------------------===//
1330// GFX6, GFX7, GFX10, GFX11.
1331//===----------------------------------------------------------------------===//
1332
1333let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1334  multiclass DS_Real_gfx6_gfx7<bits<8> op> {
1335    def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1336                                                  SIEncodingFamily.SI>;
1337  }
1338} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1339
1340multiclass DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op> :
1341  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1342
1343multiclass DS_Real_gfx6_gfx7_gfx10<bits<8> op> :
1344  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10<op>;
1345
1346defm DS_ADD_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x000>;
1347defm DS_SUB_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x001>;
1348defm DS_RSUB_U32            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x002>;
1349defm DS_INC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x003>;
1350defm DS_DEC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x004>;
1351defm DS_MIN_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x005>;
1352defm DS_MAX_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x006>;
1353defm DS_MIN_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x007>;
1354defm DS_MAX_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x008>;
1355defm DS_AND_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x009>;
1356defm DS_OR_B32              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00a>;
1357defm DS_XOR_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00b>;
1358defm DS_MSKOR_B32           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00c>;
1359
1360defm DS_WRITE_B32           : DS_Real_gfx6_gfx7_gfx10<0x00d>;
1361defm DS_WRITE2_B32          : DS_Real_gfx6_gfx7_gfx10<0x00e>;
1362defm DS_WRITE2ST64_B32      : DS_Real_gfx6_gfx7_gfx10<0x00f>;
1363defm DS_CMPST_B32           : DS_Real_gfx6_gfx7_gfx10<0x010>;
1364defm DS_CMPST_F32           : DS_Real_gfx6_gfx7_gfx10<0x011>;
1365
1366defm DS_MIN_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x012>;
1367defm DS_MAX_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x013>;
1368defm DS_NOP                 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x014>;
1369defm DS_GWS_INIT            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x019>;
1370defm DS_GWS_SEMA_V          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01a>;
1371defm DS_GWS_SEMA_BR         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01b>;
1372defm DS_GWS_SEMA_P          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01c>;
1373defm DS_GWS_BARRIER         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01d>;
1374
1375defm DS_WRITE_B8            : DS_Real_gfx6_gfx7_gfx10<0x01e>;
1376defm DS_WRITE_B16           : DS_Real_gfx6_gfx7_gfx10<0x01f>;
1377
1378defm DS_ADD_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x020>;
1379defm DS_SUB_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x021>;
1380defm DS_RSUB_RTN_U32        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x022>;
1381defm DS_INC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x023>;
1382defm DS_DEC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x024>;
1383defm DS_MIN_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x025>;
1384defm DS_MAX_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x026>;
1385defm DS_MIN_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x027>;
1386defm DS_MAX_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x028>;
1387defm DS_AND_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x029>;
1388defm DS_OR_RTN_B32          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02a>;
1389defm DS_XOR_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02b>;
1390defm DS_MSKOR_RTN_B32       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02c>;
1391
1392defm DS_WRXCHG_RTN_B32      : DS_Real_gfx6_gfx7_gfx10<0x02d>;
1393defm DS_WRXCHG2_RTN_B32     : DS_Real_gfx6_gfx7_gfx10<0x02e>;
1394defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02f>;
1395defm DS_CMPST_RTN_B32       : DS_Real_gfx6_gfx7_gfx10<0x030>;
1396defm DS_CMPST_RTN_F32       : DS_Real_gfx6_gfx7_gfx10<0x031>;
1397
1398defm DS_MIN_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x032>;
1399defm DS_MAX_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x033>;
1400defm DS_SWIZZLE_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x035>;
1401
1402defm DS_READ_B32            : DS_Real_gfx6_gfx7_gfx10<0x036>;
1403defm DS_READ2_B32           : DS_Real_gfx6_gfx7_gfx10<0x037>;
1404defm DS_READ2ST64_B32       : DS_Real_gfx6_gfx7_gfx10<0x038>;
1405defm DS_READ_I8             : DS_Real_gfx6_gfx7_gfx10<0x039>;
1406defm DS_READ_U8             : DS_Real_gfx6_gfx7_gfx10<0x03a>;
1407defm DS_READ_I16            : DS_Real_gfx6_gfx7_gfx10<0x03b>;
1408defm DS_READ_U16            : DS_Real_gfx6_gfx7_gfx10<0x03c>;
1409
1410defm DS_CONSUME             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03d>;
1411defm DS_APPEND              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03e>;
1412defm DS_ORDERED_COUNT       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03f>;
1413defm DS_ADD_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x040>;
1414defm DS_SUB_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x041>;
1415defm DS_RSUB_U64            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x042>;
1416defm DS_INC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x043>;
1417defm DS_DEC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x044>;
1418defm DS_MIN_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x045>;
1419defm DS_MAX_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x046>;
1420defm DS_MIN_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x047>;
1421defm DS_MAX_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x048>;
1422defm DS_AND_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x049>;
1423defm DS_OR_B64              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04a>;
1424defm DS_XOR_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04b>;
1425defm DS_MSKOR_B64           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04c>;
1426
1427defm DS_WRITE_B64           : DS_Real_gfx6_gfx7_gfx10<0x04d>;
1428defm DS_WRITE2_B64          : DS_Real_gfx6_gfx7_gfx10<0x04e>;
1429defm DS_WRITE2ST64_B64      : DS_Real_gfx6_gfx7_gfx10<0x04f>;
1430defm DS_CMPST_B64           : DS_Real_gfx6_gfx7_gfx10<0x050>;
1431defm DS_CMPST_F64           : DS_Real_gfx6_gfx7_gfx10<0x051>;
1432
1433defm DS_MIN_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x052>;
1434defm DS_MAX_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x053>;
1435defm DS_ADD_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x060>;
1436defm DS_SUB_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x061>;
1437defm DS_RSUB_RTN_U64        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x062>;
1438defm DS_INC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x063>;
1439defm DS_DEC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x064>;
1440defm DS_MIN_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x065>;
1441defm DS_MAX_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x066>;
1442defm DS_MIN_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x067>;
1443defm DS_MAX_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x068>;
1444defm DS_AND_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x069>;
1445defm DS_OR_RTN_B64          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06a>;
1446defm DS_XOR_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06b>;
1447defm DS_MSKOR_RTN_B64       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06c>;
1448
1449defm DS_WRXCHG_RTN_B64      : DS_Real_gfx6_gfx7_gfx10<0x06d>;
1450defm DS_WRXCHG2_RTN_B64     : DS_Real_gfx6_gfx7_gfx10<0x06e>;
1451defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06f>;
1452defm DS_CMPST_RTN_B64       : DS_Real_gfx6_gfx7_gfx10<0x070>;
1453defm DS_CMPST_RTN_F64       : DS_Real_gfx6_gfx7_gfx10<0x071>;
1454
1455defm DS_MIN_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x072>;
1456defm DS_MAX_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x073>;
1457
1458defm DS_READ_B64            : DS_Real_gfx6_gfx7_gfx10<0x076>;
1459defm DS_READ2_B64           : DS_Real_gfx6_gfx7_gfx10<0x077>;
1460defm DS_READ2ST64_B64       : DS_Real_gfx6_gfx7_gfx10<0x078>;
1461defm DS_ADD_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x080>;
1462defm DS_SUB_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x081>;
1463defm DS_RSUB_SRC2_U32       : DS_Real_gfx6_gfx7_gfx10<0x082>;
1464defm DS_INC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x083>;
1465defm DS_DEC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x084>;
1466defm DS_MIN_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x085>;
1467defm DS_MAX_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x086>;
1468defm DS_MIN_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x087>;
1469defm DS_MAX_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x088>;
1470defm DS_AND_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x089>;
1471defm DS_OR_SRC2_B32         : DS_Real_gfx6_gfx7_gfx10<0x08a>;
1472defm DS_XOR_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x08b>;
1473defm DS_WRITE_SRC2_B32      : DS_Real_gfx6_gfx7_gfx10<0x08d>;
1474defm DS_MIN_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x092>;
1475defm DS_MAX_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x093>;
1476defm DS_ADD_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c0>;
1477defm DS_SUB_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c1>;
1478defm DS_RSUB_SRC2_U64       : DS_Real_gfx6_gfx7_gfx10<0x0c2>;
1479defm DS_INC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c3>;
1480defm DS_DEC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c4>;
1481defm DS_MIN_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c5>;
1482defm DS_MAX_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c6>;
1483defm DS_MIN_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c7>;
1484defm DS_MAX_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c8>;
1485defm DS_AND_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0c9>;
1486defm DS_OR_SRC2_B64         : DS_Real_gfx6_gfx7_gfx10<0x0ca>;
1487defm DS_XOR_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0cb>;
1488defm DS_WRITE_SRC2_B64      : DS_Real_gfx6_gfx7_gfx10<0x0cd>;
1489defm DS_MIN_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d2>;
1490defm DS_MAX_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d3>;
1491
1492//===----------------------------------------------------------------------===//
1493// GFX8, GFX9 (VI).
1494//===----------------------------------------------------------------------===//
1495
1496class DS_Real_vi <bits<8> op, DS_Pseudo ps> :
1497  DS_Real <ps>,
1498  SIMCInstr <ps.Mnemonic, SIEncodingFamily.VI> {
1499  let AssemblerPredicate = isGFX8GFX9;
1500  let DecoderNamespace = "GFX8";
1501
1502  // encoding
1503  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1504  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1505  let Inst{16}    = !if(ps.has_gds, gds, ps.gdsValue);
1506  let Inst{24-17} = op;
1507  let Inst{25}    = acc;
1508  let Inst{31-26} = 0x36; // ds prefix
1509  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1510  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1511  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1512  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1513}
1514
1515def DS_ADD_U32_vi         : DS_Real_vi<0x0,  DS_ADD_U32>;
1516def DS_SUB_U32_vi         : DS_Real_vi<0x1,  DS_SUB_U32>;
1517def DS_RSUB_U32_vi        : DS_Real_vi<0x2,  DS_RSUB_U32>;
1518def DS_INC_U32_vi         : DS_Real_vi<0x3,  DS_INC_U32>;
1519def DS_DEC_U32_vi         : DS_Real_vi<0x4,  DS_DEC_U32>;
1520def DS_MIN_I32_vi         : DS_Real_vi<0x5,  DS_MIN_I32>;
1521def DS_MAX_I32_vi         : DS_Real_vi<0x6,  DS_MAX_I32>;
1522def DS_MIN_U32_vi         : DS_Real_vi<0x7,  DS_MIN_U32>;
1523def DS_MAX_U32_vi         : DS_Real_vi<0x8,  DS_MAX_U32>;
1524def DS_AND_B32_vi         : DS_Real_vi<0x9,  DS_AND_B32>;
1525def DS_OR_B32_vi          : DS_Real_vi<0xa,  DS_OR_B32>;
1526def DS_XOR_B32_vi         : DS_Real_vi<0xb,  DS_XOR_B32>;
1527def DS_MSKOR_B32_vi       : DS_Real_vi<0xc,  DS_MSKOR_B32>;
1528def DS_WRITE_B32_vi       : DS_Real_vi<0xd,  DS_WRITE_B32>;
1529def DS_WRITE2_B32_vi      : DS_Real_vi<0xe,  DS_WRITE2_B32>;
1530def DS_WRITE2ST64_B32_vi  : DS_Real_vi<0xf,  DS_WRITE2ST64_B32>;
1531def DS_CMPST_B32_vi       : DS_Real_vi<0x10, DS_CMPST_B32>;
1532def DS_CMPST_F32_vi       : DS_Real_vi<0x11, DS_CMPST_F32>;
1533def DS_MIN_F32_vi         : DS_Real_vi<0x12, DS_MIN_F32>;
1534def DS_MAX_F32_vi         : DS_Real_vi<0x13, DS_MAX_F32>;
1535def DS_NOP_vi             : DS_Real_vi<0x14, DS_NOP>;
1536def DS_ADD_F32_vi         : DS_Real_vi<0x15, DS_ADD_F32>;
1537def DS_GWS_INIT_vi        : DS_Real_vi<0x99, DS_GWS_INIT>;
1538def DS_GWS_SEMA_V_vi      : DS_Real_vi<0x9a, DS_GWS_SEMA_V>;
1539def DS_GWS_SEMA_BR_vi     : DS_Real_vi<0x9b, DS_GWS_SEMA_BR>;
1540def DS_GWS_SEMA_P_vi      : DS_Real_vi<0x9c, DS_GWS_SEMA_P>;
1541def DS_GWS_BARRIER_vi     : DS_Real_vi<0x9d, DS_GWS_BARRIER>;
1542def DS_WRITE_ADDTID_B32_vi : DS_Real_vi<0x1d, DS_WRITE_ADDTID_B32>;
1543def DS_WRITE_B8_vi        : DS_Real_vi<0x1e, DS_WRITE_B8>;
1544def DS_WRITE_B16_vi       : DS_Real_vi<0x1f, DS_WRITE_B16>;
1545def DS_ADD_RTN_U32_vi     : DS_Real_vi<0x20, DS_ADD_RTN_U32>;
1546def DS_SUB_RTN_U32_vi     : DS_Real_vi<0x21, DS_SUB_RTN_U32>;
1547def DS_RSUB_RTN_U32_vi    : DS_Real_vi<0x22, DS_RSUB_RTN_U32>;
1548def DS_INC_RTN_U32_vi     : DS_Real_vi<0x23, DS_INC_RTN_U32>;
1549def DS_DEC_RTN_U32_vi     : DS_Real_vi<0x24, DS_DEC_RTN_U32>;
1550def DS_MIN_RTN_I32_vi     : DS_Real_vi<0x25, DS_MIN_RTN_I32>;
1551def DS_MAX_RTN_I32_vi     : DS_Real_vi<0x26, DS_MAX_RTN_I32>;
1552def DS_MIN_RTN_U32_vi     : DS_Real_vi<0x27, DS_MIN_RTN_U32>;
1553def DS_MAX_RTN_U32_vi     : DS_Real_vi<0x28, DS_MAX_RTN_U32>;
1554def DS_AND_RTN_B32_vi     : DS_Real_vi<0x29, DS_AND_RTN_B32>;
1555def DS_OR_RTN_B32_vi      : DS_Real_vi<0x2a, DS_OR_RTN_B32>;
1556def DS_XOR_RTN_B32_vi     : DS_Real_vi<0x2b, DS_XOR_RTN_B32>;
1557def DS_MSKOR_RTN_B32_vi   : DS_Real_vi<0x2c, DS_MSKOR_RTN_B32>;
1558def DS_WRXCHG_RTN_B32_vi  : DS_Real_vi<0x2d, DS_WRXCHG_RTN_B32>;
1559def DS_WRXCHG2_RTN_B32_vi : DS_Real_vi<0x2e, DS_WRXCHG2_RTN_B32>;
1560def DS_WRXCHG2ST64_RTN_B32_vi : DS_Real_vi<0x2f, DS_WRXCHG2ST64_RTN_B32>;
1561def DS_CMPST_RTN_B32_vi   : DS_Real_vi<0x30, DS_CMPST_RTN_B32>;
1562def DS_CMPST_RTN_F32_vi   : DS_Real_vi<0x31, DS_CMPST_RTN_F32>;
1563def DS_MIN_RTN_F32_vi     : DS_Real_vi<0x32, DS_MIN_RTN_F32>;
1564def DS_MAX_RTN_F32_vi     : DS_Real_vi<0x33, DS_MAX_RTN_F32>;
1565def DS_WRAP_RTN_B32_vi    : DS_Real_vi<0x34, DS_WRAP_RTN_B32>;
1566def DS_ADD_RTN_F32_vi     : DS_Real_vi<0x35, DS_ADD_RTN_F32>;
1567def DS_READ_B32_vi        : DS_Real_vi<0x36, DS_READ_B32>;
1568def DS_READ2_B32_vi       : DS_Real_vi<0x37, DS_READ2_B32>;
1569def DS_READ2ST64_B32_vi   : DS_Real_vi<0x38, DS_READ2ST64_B32>;
1570def DS_READ_I8_vi         : DS_Real_vi<0x39, DS_READ_I8>;
1571def DS_READ_U8_vi         : DS_Real_vi<0x3a, DS_READ_U8>;
1572def DS_READ_I16_vi        : DS_Real_vi<0x3b, DS_READ_I16>;
1573def DS_READ_U16_vi        : DS_Real_vi<0x3c, DS_READ_U16>;
1574def DS_READ_ADDTID_B32_vi : DS_Real_vi<0xb6, DS_READ_ADDTID_B32>;
1575def DS_CONSUME_vi         : DS_Real_vi<0xbd, DS_CONSUME>;
1576def DS_APPEND_vi          : DS_Real_vi<0xbe, DS_APPEND>;
1577def DS_ORDERED_COUNT_vi   : DS_Real_vi<0xbf, DS_ORDERED_COUNT>;
1578def DS_SWIZZLE_B32_vi     : DS_Real_vi<0x3d, DS_SWIZZLE_B32>;
1579def DS_PERMUTE_B32_vi     : DS_Real_vi<0x3e, DS_PERMUTE_B32>;
1580def DS_BPERMUTE_B32_vi    : DS_Real_vi<0x3f, DS_BPERMUTE_B32>;
1581
1582def DS_ADD_U64_vi         : DS_Real_vi<0x40, DS_ADD_U64>;
1583def DS_SUB_U64_vi         : DS_Real_vi<0x41, DS_SUB_U64>;
1584def DS_RSUB_U64_vi        : DS_Real_vi<0x42, DS_RSUB_U64>;
1585def DS_INC_U64_vi         : DS_Real_vi<0x43, DS_INC_U64>;
1586def DS_DEC_U64_vi         : DS_Real_vi<0x44, DS_DEC_U64>;
1587def DS_MIN_I64_vi         : DS_Real_vi<0x45, DS_MIN_I64>;
1588def DS_MAX_I64_vi         : DS_Real_vi<0x46, DS_MAX_I64>;
1589def DS_MIN_U64_vi         : DS_Real_vi<0x47, DS_MIN_U64>;
1590def DS_MAX_U64_vi         : DS_Real_vi<0x48, DS_MAX_U64>;
1591def DS_AND_B64_vi         : DS_Real_vi<0x49, DS_AND_B64>;
1592def DS_OR_B64_vi          : DS_Real_vi<0x4a, DS_OR_B64>;
1593def DS_XOR_B64_vi         : DS_Real_vi<0x4b, DS_XOR_B64>;
1594def DS_MSKOR_B64_vi       : DS_Real_vi<0x4c, DS_MSKOR_B64>;
1595def DS_WRITE_B64_vi       : DS_Real_vi<0x4d, DS_WRITE_B64>;
1596def DS_WRITE2_B64_vi      : DS_Real_vi<0x4E, DS_WRITE2_B64>;
1597def DS_WRITE2ST64_B64_vi  : DS_Real_vi<0x4f, DS_WRITE2ST64_B64>;
1598def DS_CMPST_B64_vi       : DS_Real_vi<0x50, DS_CMPST_B64>;
1599def DS_CMPST_F64_vi       : DS_Real_vi<0x51, DS_CMPST_F64>;
1600def DS_MIN_F64_vi         : DS_Real_vi<0x52, DS_MIN_F64>;
1601def DS_MAX_F64_vi         : DS_Real_vi<0x53, DS_MAX_F64>;
1602
1603def DS_WRITE_B8_D16_HI_vi  : DS_Real_vi<0x54, DS_WRITE_B8_D16_HI>;
1604def DS_WRITE_B16_D16_HI_vi : DS_Real_vi<0x55, DS_WRITE_B16_D16_HI>;
1605
1606def DS_READ_U8_D16_vi     : DS_Real_vi<0x56, DS_READ_U8_D16>;
1607def DS_READ_U8_D16_HI_vi  : DS_Real_vi<0x57, DS_READ_U8_D16_HI>;
1608def DS_READ_I8_D16_vi     : DS_Real_vi<0x58, DS_READ_I8_D16>;
1609def DS_READ_I8_D16_HI_vi  : DS_Real_vi<0x59, DS_READ_I8_D16_HI>;
1610def DS_READ_U16_D16_vi    : DS_Real_vi<0x5a, DS_READ_U16_D16>;
1611def DS_READ_U16_D16_HI_vi : DS_Real_vi<0x5b, DS_READ_U16_D16_HI>;
1612
1613def DS_ADD_RTN_U64_vi     : DS_Real_vi<0x60, DS_ADD_RTN_U64>;
1614def DS_SUB_RTN_U64_vi     : DS_Real_vi<0x61, DS_SUB_RTN_U64>;
1615def DS_RSUB_RTN_U64_vi    : DS_Real_vi<0x62, DS_RSUB_RTN_U64>;
1616def DS_INC_RTN_U64_vi     : DS_Real_vi<0x63, DS_INC_RTN_U64>;
1617def DS_DEC_RTN_U64_vi     : DS_Real_vi<0x64, DS_DEC_RTN_U64>;
1618def DS_MIN_RTN_I64_vi     : DS_Real_vi<0x65, DS_MIN_RTN_I64>;
1619def DS_MAX_RTN_I64_vi     : DS_Real_vi<0x66, DS_MAX_RTN_I64>;
1620def DS_MIN_RTN_U64_vi     : DS_Real_vi<0x67, DS_MIN_RTN_U64>;
1621def DS_MAX_RTN_U64_vi     : DS_Real_vi<0x68, DS_MAX_RTN_U64>;
1622def DS_AND_RTN_B64_vi     : DS_Real_vi<0x69, DS_AND_RTN_B64>;
1623def DS_OR_RTN_B64_vi      : DS_Real_vi<0x6a, DS_OR_RTN_B64>;
1624def DS_XOR_RTN_B64_vi     : DS_Real_vi<0x6b, DS_XOR_RTN_B64>;
1625def DS_MSKOR_RTN_B64_vi   : DS_Real_vi<0x6c, DS_MSKOR_RTN_B64>;
1626def DS_WRXCHG_RTN_B64_vi  : DS_Real_vi<0x6d, DS_WRXCHG_RTN_B64>;
1627def DS_WRXCHG2_RTN_B64_vi : DS_Real_vi<0x6e, DS_WRXCHG2_RTN_B64>;
1628def DS_WRXCHG2ST64_RTN_B64_vi : DS_Real_vi<0x6f, DS_WRXCHG2ST64_RTN_B64>;
1629def DS_CONDXCHG32_RTN_B64_vi   : DS_Real_vi<0x7e, DS_CONDXCHG32_RTN_B64>;
1630def DS_GWS_SEMA_RELEASE_ALL_vi : DS_Real_vi<0x98, DS_GWS_SEMA_RELEASE_ALL>;
1631def DS_CMPST_RTN_B64_vi   : DS_Real_vi<0x70, DS_CMPST_RTN_B64>;
1632def DS_CMPST_RTN_F64_vi   : DS_Real_vi<0x71, DS_CMPST_RTN_F64>;
1633def DS_MIN_RTN_F64_vi     : DS_Real_vi<0x72, DS_MIN_RTN_F64>;
1634def DS_MAX_RTN_F64_vi     : DS_Real_vi<0x73, DS_MAX_RTN_F64>;
1635
1636def DS_READ_B64_vi        : DS_Real_vi<0x76, DS_READ_B64>;
1637def DS_READ2_B64_vi       : DS_Real_vi<0x77, DS_READ2_B64>;
1638def DS_READ2ST64_B64_vi   : DS_Real_vi<0x78, DS_READ2ST64_B64>;
1639
1640def DS_ADD_SRC2_U32_vi    : DS_Real_vi<0x80, DS_ADD_SRC2_U32>;
1641def DS_SUB_SRC2_U32_vi    : DS_Real_vi<0x81, DS_SUB_SRC2_U32>;
1642def DS_RSUB_SRC2_U32_vi   : DS_Real_vi<0x82, DS_RSUB_SRC2_U32>;
1643def DS_INC_SRC2_U32_vi    : DS_Real_vi<0x83, DS_INC_SRC2_U32>;
1644def DS_DEC_SRC2_U32_vi    : DS_Real_vi<0x84, DS_DEC_SRC2_U32>;
1645def DS_MIN_SRC2_I32_vi    : DS_Real_vi<0x85, DS_MIN_SRC2_I32>;
1646def DS_MAX_SRC2_I32_vi    : DS_Real_vi<0x86, DS_MAX_SRC2_I32>;
1647def DS_MIN_SRC2_U32_vi    : DS_Real_vi<0x87, DS_MIN_SRC2_U32>;
1648def DS_MAX_SRC2_U32_vi    : DS_Real_vi<0x88, DS_MAX_SRC2_U32>;
1649def DS_AND_SRC2_B32_vi    : DS_Real_vi<0x89, DS_AND_SRC2_B32>;
1650def DS_OR_SRC2_B32_vi     : DS_Real_vi<0x8a, DS_OR_SRC2_B32>;
1651def DS_XOR_SRC2_B32_vi    : DS_Real_vi<0x8b, DS_XOR_SRC2_B32>;
1652def DS_WRITE_SRC2_B32_vi  : DS_Real_vi<0x8d, DS_WRITE_SRC2_B32>;
1653def DS_MIN_SRC2_F32_vi    : DS_Real_vi<0x92, DS_MIN_SRC2_F32>;
1654def DS_MAX_SRC2_F32_vi    : DS_Real_vi<0x93, DS_MAX_SRC2_F32>;
1655def DS_ADD_SRC2_F32_vi    : DS_Real_vi<0x95, DS_ADD_SRC2_F32>;
1656def DS_ADD_SRC2_U64_vi    : DS_Real_vi<0xc0, DS_ADD_SRC2_U64>;
1657def DS_SUB_SRC2_U64_vi    : DS_Real_vi<0xc1, DS_SUB_SRC2_U64>;
1658def DS_RSUB_SRC2_U64_vi   : DS_Real_vi<0xc2, DS_RSUB_SRC2_U64>;
1659def DS_INC_SRC2_U64_vi    : DS_Real_vi<0xc3, DS_INC_SRC2_U64>;
1660def DS_DEC_SRC2_U64_vi    : DS_Real_vi<0xc4, DS_DEC_SRC2_U64>;
1661def DS_MIN_SRC2_I64_vi    : DS_Real_vi<0xc5, DS_MIN_SRC2_I64>;
1662def DS_MAX_SRC2_I64_vi    : DS_Real_vi<0xc6, DS_MAX_SRC2_I64>;
1663def DS_MIN_SRC2_U64_vi    : DS_Real_vi<0xc7, DS_MIN_SRC2_U64>;
1664def DS_MAX_SRC2_U64_vi    : DS_Real_vi<0xc8, DS_MAX_SRC2_U64>;
1665def DS_AND_SRC2_B64_vi    : DS_Real_vi<0xc9, DS_AND_SRC2_B64>;
1666def DS_OR_SRC2_B64_vi     : DS_Real_vi<0xca, DS_OR_SRC2_B64>;
1667def DS_XOR_SRC2_B64_vi    : DS_Real_vi<0xcb, DS_XOR_SRC2_B64>;
1668def DS_WRITE_SRC2_B64_vi  : DS_Real_vi<0xcd, DS_WRITE_SRC2_B64>;
1669def DS_MIN_SRC2_F64_vi    : DS_Real_vi<0xd2, DS_MIN_SRC2_F64>;
1670def DS_MAX_SRC2_F64_vi    : DS_Real_vi<0xd3, DS_MAX_SRC2_F64>;
1671def DS_WRITE_B96_vi       : DS_Real_vi<0xde, DS_WRITE_B96>;
1672def DS_WRITE_B128_vi      : DS_Real_vi<0xdf, DS_WRITE_B128>;
1673def DS_READ_B96_vi        : DS_Real_vi<0xfe, DS_READ_B96>;
1674def DS_READ_B128_vi       : DS_Real_vi<0xff, DS_READ_B128>;
1675
1676let SubtargetPredicate = isGFX90APlus in {
1677  def DS_ADD_F64_vi     : DS_Real_vi<0x5c, DS_ADD_F64>;
1678  def DS_ADD_RTN_F64_vi : DS_Real_vi<0x7c, DS_ADD_RTN_F64>;
1679} // End SubtargetPredicate = isGFX90APlus
1680
1681let SubtargetPredicate = isGFX940Plus in {
1682  def DS_PK_ADD_F16_vi     : DS_Real_vi<0x17, DS_PK_ADD_F16>;
1683  def DS_PK_ADD_RTN_F16_vi : DS_Real_vi<0xb7, DS_PK_ADD_RTN_F16>;
1684  def DS_PK_ADD_BF16_vi     : DS_Real_vi<0x18, DS_PK_ADD_BF16>;
1685  def DS_PK_ADD_RTN_BF16_vi : DS_Real_vi<0xb8, DS_PK_ADD_RTN_BF16>;
1686} // End SubtargetPredicate = isGFX940Plus
1687