xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/DSInstructions.td (revision 972a253a57b6f144b0e4a3e2080a2a0076ec55a0) 
1//===-- DSInstructions.td - DS Instruction Definitions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
10  InstSI <outs, ins, "", pattern>,
11  SIMCInstr <opName, SIEncodingFamily.NONE> {
12
13  let LGKM_CNT = 1;
14  let DS = 1;
15  let Size = 8;
16  let UseNamedOperandTable = 1;
17
18  // Most instruction load and store data, so set this as the default.
19  let mayLoad = 1;
20  let mayStore = 1;
21  let maybeAtomic = 1;
22
23  let hasSideEffects = 0;
24  let SchedRW = [WriteLDS];
25
26  let isPseudo = 1;
27  let isCodeGenOnly = 1;
28
29  let AsmMatchConverter = "cvtDS";
30
31  string Mnemonic = opName;
32  string AsmOperands = asmOps;
33
34  // Well these bits a kind of hack because it would be more natural
35  // to test "outs" and "ins" dags for the presence of particular operands
36  bits<1> has_vdst = 1;
37  bits<1> has_addr = 1;
38  bits<1> has_data0 = 1;
39  bits<1> has_data1 = 1;
40
41  bits<1> has_gws_data0 = 0; // data0 is encoded as addr
42
43  bits<1> has_offset  = 1; // has "offset" that should be split to offset0,1
44  bits<1> has_offset0 = 1;
45  bits<1> has_offset1 = 1;
46
47  bits<1> has_gds = 1;
48  bits<1> gdsValue = 0; // if has_gds == 0 set gds to this value
49
50  bits<1> has_m0_read = 1;
51
52  let Uses = !if(has_m0_read, [M0, EXEC], [EXEC]);
53}
54
55class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
56  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands>,
57  Enc64 {
58
59  let isPseudo = 0;
60  let isCodeGenOnly = 0;
61  let LGKM_CNT = 1;
62  let DS = 1;
63  let UseNamedOperandTable = 1;
64
65  // copy relevant pseudo op flags
66  let SubtargetPredicate = ps.SubtargetPredicate;
67  let OtherPredicates    = ps.OtherPredicates;
68  let AsmMatchConverter  = ps.AsmMatchConverter;
69  let SchedRW            = ps.SchedRW;
70  let mayLoad            = ps.mayLoad;
71  let mayStore           = ps.mayStore;
72  let IsAtomicRet        = ps.IsAtomicRet;
73  let IsAtomicNoRet      = ps.IsAtomicNoRet;
74
75  let Constraints = ps.Constraints;
76  let DisableEncoding = ps.DisableEncoding;
77
78  // encoding fields
79  bits<10> vdst;
80  bits<1> gds;
81  bits<8> addr;
82  bits<10> data0;
83  bits<10> data1;
84  bits<8> offset0;
85  bits<8> offset1;
86
87  bits<16> offset;
88  let offset0 = !if(ps.has_offset, offset{7-0}, ?);
89  let offset1 = !if(ps.has_offset, offset{15-8}, ?);
90
91  bits<1> acc = !if(ps.has_vdst, vdst{9},
92                    !if(!or(ps.has_data0, ps.has_gws_data0), data0{9}, 0));
93}
94
95// DS Pseudo instructions
96
97class DS_0A1D_NORET<string opName, RegisterClass rc = VGPR_32>
98: DS_Pseudo<opName,
99  (outs),
100  (ins getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
101  " $data0$offset$gds"> {
102
103  let has_addr = 0;
104  let has_data1 = 0;
105  let has_vdst = 0;
106}
107
108class DS_1A1D_NORET<string opName, RegisterClass rc = VGPR_32>
109: DS_Pseudo<opName,
110  (outs),
111  (ins VGPR_32:$addr, getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
112  " $addr, $data0$offset$gds"> {
113
114  let has_data1 = 0;
115  let has_vdst = 0;
116  let IsAtomicNoRet = 1;
117}
118
119multiclass DS_1A1D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
120  def "" : DS_1A1D_NORET<opName, rc>,
121           AtomicNoRet<opName, 0>;
122
123  let has_m0_read = 0 in {
124    def _gfx9 : DS_1A1D_NORET<opName, rc>,
125                AtomicNoRet<opName#"_gfx9", 0>;
126  }
127}
128
129multiclass DS_1A1D_NORET_mc_gfx9<string opName, RegisterClass rc = VGPR_32> {
130  let has_m0_read = 0 in {
131    def "" : DS_1A1D_NORET<opName, rc>,
132                AtomicNoRet<opName, 0>;
133  }
134}
135
136class DS_1A2D_NORET<string opName, RegisterClass rc = VGPR_32,
137                    RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
138: DS_Pseudo<opName,
139  (outs),
140  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1, offset:$offset, gds:$gds),
141  " $addr, $data0, $data1$offset$gds"> {
142
143  let has_vdst = 0;
144  let IsAtomicNoRet = 1;
145}
146
147multiclass DS_1A2D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
148  def "" : DS_1A2D_NORET<opName, rc>,
149           AtomicNoRet<opName, 0>;
150
151  let has_m0_read = 0 in {
152    def _gfx9 : DS_1A2D_NORET<opName, rc>,
153                AtomicNoRet<opName#"_gfx9", 0>;
154  }
155}
156
157class DS_1A2D_Off8_NORET <string opName, RegisterClass rc = VGPR_32,
158                          RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
159: DS_Pseudo<opName,
160  (outs),
161  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1,
162       offset0:$offset0, offset1:$offset1, gds:$gds),
163  " $addr, $data0, $data1$offset0$offset1$gds"> {
164
165  let has_vdst = 0;
166  let has_offset = 0;
167  let AsmMatchConverter = "cvtDSOffset01";
168}
169
170multiclass DS_1A2D_Off8_NORET_mc <string opName, RegisterClass rc = VGPR_32> {
171  def "" : DS_1A2D_Off8_NORET<opName, rc>;
172
173  let has_m0_read = 0 in {
174    def _gfx9 : DS_1A2D_Off8_NORET<opName, rc>;
175  }
176}
177
178class DS_0A1D_RET_GDS<string opName, RegisterClass rc = VGPR_32, RegisterClass src = rc,
179                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
180                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
181: DS_Pseudo<opName,
182  (outs dst_op:$vdst),
183  (ins src_op:$data0, offset:$offset),
184  " $vdst, $data0$offset gds"> {
185
186  let has_addr = 0;
187  let has_data1 = 0;
188  let has_gds = 0;
189  let gdsValue = 1;
190  let AsmMatchConverter = "cvtDSGds";
191  let hasSideEffects = 1;
192}
193
194class DS_1A1D_RET <string opName, RegisterClass rc = VGPR_32,
195                  RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
196: DS_Pseudo<opName,
197  (outs data_op:$vdst),
198  (ins VGPR_32:$addr, data_op:$data0, offset:$offset, gds:$gds),
199  " $vdst, $addr, $data0$offset$gds"> {
200
201  let hasPostISelHook = 1;
202  let has_data1 = 0;
203  let IsAtomicRet = 1;
204}
205
206multiclass DS_1A1D_RET_mc <string opName, RegisterClass rc = VGPR_32,
207                           string NoRetOp = ""> {
208  def "" : DS_1A1D_RET<opName, rc>,
209    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
210
211  let has_m0_read = 0 in {
212    def _gfx9 : DS_1A1D_RET<opName, rc>,
213      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp#"_gfx9"),
214                  !ne(NoRetOp, "")>;
215  }
216}
217
218multiclass DS_1A1D_RET_mc_gfx9 <string opName, RegisterClass rc = VGPR_32,
219                                string NoRetOp = ""> {
220  let has_m0_read = 0 in {
221    def "" : DS_1A1D_RET<opName, rc>,
222      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp),
223                  !if(!eq(NoRetOp, ""), 0, 1)>;
224  }
225}
226
227class DS_1A2D_RET<string opName,
228                  RegisterClass rc = VGPR_32,
229                  RegisterClass src = rc,
230                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
231                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
232: DS_Pseudo<opName,
233  (outs dst_op:$vdst),
234  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset:$offset, gds:$gds),
235  " $vdst, $addr, $data0, $data1$offset$gds"> {
236
237  let hasPostISelHook = 1;
238  let IsAtomicRet = 1;
239}
240
241multiclass DS_1A2D_RET_mc<string opName,
242                          RegisterClass rc = VGPR_32,
243                          string NoRetOp = "",
244                          RegisterClass src = rc> {
245  def "" : DS_1A2D_RET<opName, rc, src>,
246    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
247
248  let has_m0_read = 0 in {
249    def _gfx9 : DS_1A2D_RET<opName, rc, src>,
250      AtomicNoRet<NoRetOp#"_gfx9", !ne(NoRetOp, "")>;
251  }
252}
253
254class DS_1A2D_Off8_RET<string opName,
255                       RegisterClass rc = VGPR_32,
256                       RegisterClass src = rc,
257                       RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
258                       RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
259: DS_Pseudo<opName,
260  (outs dst_op:$vdst),
261  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset0:$offset0, offset1:$offset1, gds:$gds),
262  " $vdst, $addr, $data0, $data1$offset0$offset1$gds"> {
263
264  let has_offset = 0;
265  let AsmMatchConverter = "cvtDSOffset01";
266
267  let hasPostISelHook = 1;
268}
269
270multiclass DS_1A2D_Off8_RET_mc<string opName,
271                               RegisterClass rc = VGPR_32,
272                               RegisterClass src = rc> {
273  def "" : DS_1A2D_Off8_RET<opName, rc, src>;
274
275  let has_m0_read = 0 in {
276    def _gfx9 : DS_1A2D_Off8_RET<opName, rc, src>;
277  }
278}
279
280
281class DS_1A_RET<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset,
282                RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
283: DS_Pseudo<opName,
284  (outs data_op:$vdst),
285  !if(HasTiedOutput,
286    (ins VGPR_32:$addr, ofs:$offset, gds:$gds, data_op:$vdst_in),
287    (ins VGPR_32:$addr, ofs:$offset, gds:$gds)),
288  " $vdst, $addr$offset$gds"> {
289  let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");
290  let DisableEncoding = !if(HasTiedOutput, "$vdst_in", "");
291  let has_data0 = 0;
292  let has_data1 = 0;
293}
294
295multiclass DS_1A_RET_mc<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset> {
296  def "" : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
297
298  let has_m0_read = 0 in {
299    def _gfx9 : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
300  }
301}
302
303class DS_1A_RET_Tied<string opName, RegisterClass rc = VGPR_32> :
304  DS_1A_RET<opName, rc, 1>;
305
306class DS_1A_Off8_RET <string opName, RegisterClass rc = VGPR_32>
307: DS_Pseudo<opName,
308  (outs getLdStRegisterOperand<rc>.ret:$vdst),
309  (ins VGPR_32:$addr, offset0:$offset0, offset1:$offset1, gds:$gds),
310  " $vdst, $addr$offset0$offset1$gds"> {
311
312  let has_offset = 0;
313  let has_data0 = 0;
314  let has_data1 = 0;
315  let AsmMatchConverter = "cvtDSOffset01";
316}
317
318multiclass DS_1A_Off8_RET_mc <string opName, RegisterClass rc = VGPR_32> {
319  def "" : DS_1A_Off8_RET<opName, rc>;
320
321  let has_m0_read = 0 in {
322    def _gfx9 : DS_1A_Off8_RET<opName, rc>;
323  }
324}
325
326class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,
327  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
328  (ins VGPR_32:$addr, offset:$offset),
329  " $vdst, $addr$offset gds"> {
330
331  let has_data0 = 0;
332  let has_data1 = 0;
333  let has_gds = 0;
334  let gdsValue = 1;
335  let AsmMatchConverter = "cvtDSGds";
336}
337
338class DS_0A_RET <string opName> : DS_Pseudo<opName,
339  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
340  (ins offset:$offset, gds:$gds),
341  " $vdst$offset$gds"> {
342
343  let mayLoad = 1;
344  let mayStore = 1;
345
346  let has_addr = 0;
347  let has_data0 = 0;
348  let has_data1 = 0;
349}
350
351class DS_1A <string opName> : DS_Pseudo<opName,
352  (outs),
353  (ins VGPR_32:$addr, offset:$offset, gds:$gds),
354  " $addr$offset$gds"> {
355
356  let mayLoad = 1;
357  let mayStore = 1;
358
359  let has_vdst = 0;
360  let has_data0 = 0;
361  let has_data1 = 0;
362}
363
364multiclass DS_1A_mc <string opName> {
365  def "" : DS_1A<opName>;
366
367  let has_m0_read = 0 in {
368    def _gfx9 : DS_1A<opName>;
369  }
370}
371
372
373class DS_GWS <string opName, dag ins, string asmOps>
374: DS_Pseudo<opName, (outs), ins, asmOps> {
375
376  let has_vdst  = 0;
377  let has_addr  = 0;
378  let has_data0 = 0;
379  let has_data1 = 0;
380
381  let has_gds   = 0;
382  let gdsValue  = 1;
383  let AsmMatchConverter = "cvtDSGds";
384}
385
386class DS_GWS_0D <string opName>
387: DS_GWS<opName,
388  (ins offset:$offset), "$offset gds"> {
389  let hasSideEffects = 1;
390}
391
392class DS_GWS_1D <string opName>
393: DS_GWS<opName,
394  (ins getLdStRegisterOperand<VGPR_32>.ret:$data0, offset:$offset),
395  " $data0$offset gds"> {
396
397  let has_gws_data0 = 1;
398  let hasSideEffects = 1;
399}
400
401class DS_VOID <string opName> : DS_Pseudo<opName,
402  (outs), (ins), ""> {
403  let mayLoad = 0;
404  let mayStore = 0;
405  let hasSideEffects = 1;
406  let UseNamedOperandTable = 0;
407  let AsmMatchConverter = "";
408
409  let has_vdst = 0;
410  let has_addr = 0;
411  let has_data0 = 0;
412  let has_data1 = 0;
413  let has_offset = 0;
414  let has_offset0 = 0;
415  let has_offset1 = 0;
416  let has_gds = 0;
417}
418
419class DS_1A1D_PERMUTE <string opName, SDPatternOperator node = null_frag,
420                       RegisterOperand data_op = getLdStRegisterOperand<VGPR_32>.ret>
421: DS_Pseudo<opName,
422  (outs data_op:$vdst),
423  (ins VGPR_32:$addr, data_op:$data0, offset:$offset),
424  " $vdst, $addr, $data0$offset",
425  [(set i32:$vdst,
426   (node (DS1Addr1Offset i32:$addr, i16:$offset), i32:$data0))] > {
427
428  let mayLoad = 0;
429  let mayStore = 0;
430  let isConvergent = 1;
431
432  let has_data1 = 0;
433  let has_gds = 0;
434}
435
436defm DS_ADD_U32       : DS_1A1D_NORET_mc<"ds_add_u32">;
437defm DS_SUB_U32       : DS_1A1D_NORET_mc<"ds_sub_u32">;
438defm DS_RSUB_U32      : DS_1A1D_NORET_mc<"ds_rsub_u32">;
439defm DS_INC_U32       : DS_1A1D_NORET_mc<"ds_inc_u32">;
440defm DS_DEC_U32       : DS_1A1D_NORET_mc<"ds_dec_u32">;
441defm DS_MIN_I32       : DS_1A1D_NORET_mc<"ds_min_i32">;
442defm DS_MAX_I32       : DS_1A1D_NORET_mc<"ds_max_i32">;
443defm DS_MIN_U32       : DS_1A1D_NORET_mc<"ds_min_u32">;
444defm DS_MAX_U32       : DS_1A1D_NORET_mc<"ds_max_u32">;
445defm DS_AND_B32       : DS_1A1D_NORET_mc<"ds_and_b32">;
446defm DS_OR_B32        : DS_1A1D_NORET_mc<"ds_or_b32">;
447defm DS_XOR_B32       : DS_1A1D_NORET_mc<"ds_xor_b32">;
448
449let SubtargetPredicate = HasLDSFPAtomicAdd in {
450defm DS_ADD_F32       : DS_1A1D_NORET_mc<"ds_add_f32">;
451}
452
453defm DS_MIN_F32       : DS_1A1D_NORET_mc<"ds_min_f32">;
454defm DS_MAX_F32       : DS_1A1D_NORET_mc<"ds_max_f32">;
455
456let mayLoad = 0 in {
457defm DS_WRITE_B8      : DS_1A1D_NORET_mc<"ds_write_b8">;
458defm DS_WRITE_B16     : DS_1A1D_NORET_mc<"ds_write_b16">;
459defm DS_WRITE_B32     : DS_1A1D_NORET_mc<"ds_write_b32">;
460defm DS_WRITE2_B32    : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">;
461defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">;
462
463
464let has_m0_read = 0 in {
465
466let SubtargetPredicate = HasD16LoadStore in {
467def DS_WRITE_B8_D16_HI  : DS_1A1D_NORET<"ds_write_b8_d16_hi">;
468def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;
469}
470
471} // End has_m0_read = 0
472
473let SubtargetPredicate = HasDSAddTid in {
474def DS_WRITE_ADDTID_B32 : DS_0A1D_NORET<"ds_write_addtid_b32">;
475}
476
477} // End mayLoad = 0
478
479let SubtargetPredicate = isGFX90APlus in {
480  defm DS_ADD_F64     : DS_1A1D_NORET_mc_gfx9<"ds_add_f64", VReg_64>;
481  defm DS_ADD_RTN_F64 : DS_1A1D_RET_mc_gfx9<"ds_add_rtn_f64", VReg_64, "ds_add_f64">;
482} // End SubtargetPredicate = isGFX90APlus
483
484let SubtargetPredicate = isGFX940Plus in {
485  defm DS_PK_ADD_F16      : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_f16">;
486  defm DS_PK_ADD_RTN_F16  : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_f16", VGPR_32, "ds_pk_add_f16">;
487  defm DS_PK_ADD_BF16     : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_bf16">;
488  defm DS_PK_ADD_RTN_BF16 : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_bf16", VGPR_32, "ds_pk_add_bf16">;
489} // End SubtargetPredicate = isGFX940Plus
490
491defm DS_CMPSTORE_B32     : DS_1A2D_NORET_mc<"ds_cmpstore_b32">;
492defm DS_CMPSTORE_F32     : DS_1A2D_NORET_mc<"ds_cmpstore_f32">;
493defm DS_CMPSTORE_B64     : DS_1A2D_NORET_mc<"ds_cmpstore_b64", VReg_64>;
494defm DS_CMPSTORE_F64     : DS_1A2D_NORET_mc<"ds_cmpstore_f64", VReg_64>;
495defm DS_CMPSTORE_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b32", VGPR_32, "ds_cmpstore_b32">;
496defm DS_CMPSTORE_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f32", VGPR_32, "ds_cmpstore_f32">;
497defm DS_CMPSTORE_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b64", VReg_64, "ds_cmpstore_b64">;
498defm DS_CMPSTORE_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f64", VReg_64, "ds_cmpstore_f64">;
499
500defm DS_MSKOR_B32     : DS_1A2D_NORET_mc<"ds_mskor_b32">;
501defm DS_CMPST_B32     : DS_1A2D_NORET_mc<"ds_cmpst_b32">;
502defm DS_CMPST_F32     : DS_1A2D_NORET_mc<"ds_cmpst_f32">;
503
504defm DS_ADD_U64       : DS_1A1D_NORET_mc<"ds_add_u64", VReg_64>;
505defm DS_SUB_U64       : DS_1A1D_NORET_mc<"ds_sub_u64", VReg_64>;
506defm DS_RSUB_U64      : DS_1A1D_NORET_mc<"ds_rsub_u64", VReg_64>;
507defm DS_INC_U64       : DS_1A1D_NORET_mc<"ds_inc_u64", VReg_64>;
508defm DS_DEC_U64       : DS_1A1D_NORET_mc<"ds_dec_u64", VReg_64>;
509defm DS_MIN_I64       : DS_1A1D_NORET_mc<"ds_min_i64", VReg_64>;
510defm DS_MAX_I64       : DS_1A1D_NORET_mc<"ds_max_i64", VReg_64>;
511defm DS_MIN_U64       : DS_1A1D_NORET_mc<"ds_min_u64", VReg_64>;
512defm DS_MAX_U64       : DS_1A1D_NORET_mc<"ds_max_u64", VReg_64>;
513defm DS_AND_B64       : DS_1A1D_NORET_mc<"ds_and_b64", VReg_64>;
514defm DS_OR_B64        : DS_1A1D_NORET_mc<"ds_or_b64", VReg_64>;
515defm DS_XOR_B64       : DS_1A1D_NORET_mc<"ds_xor_b64", VReg_64>;
516defm DS_MSKOR_B64     : DS_1A2D_NORET_mc<"ds_mskor_b64", VReg_64>;
517let mayLoad = 0 in {
518defm DS_WRITE_B64     : DS_1A1D_NORET_mc<"ds_write_b64", VReg_64>;
519defm DS_WRITE2_B64    : DS_1A2D_Off8_NORET_mc<"ds_write2_b64", VReg_64>;
520defm DS_WRITE2ST64_B64: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b64", VReg_64>;
521}
522defm DS_CMPST_B64     : DS_1A2D_NORET_mc<"ds_cmpst_b64", VReg_64>;
523defm DS_CMPST_F64     : DS_1A2D_NORET_mc<"ds_cmpst_f64", VReg_64>;
524defm DS_MIN_F64       : DS_1A1D_NORET_mc<"ds_min_f64", VReg_64>;
525defm DS_MAX_F64       : DS_1A1D_NORET_mc<"ds_max_f64", VReg_64>;
526
527defm DS_ADD_RTN_U32   : DS_1A1D_RET_mc<"ds_add_rtn_u32", VGPR_32, "ds_add_u32">;
528
529let SubtargetPredicate = HasLDSFPAtomicAdd in {
530defm DS_ADD_RTN_F32   : DS_1A1D_RET_mc<"ds_add_rtn_f32", VGPR_32, "ds_add_f32">;
531}
532defm DS_SUB_RTN_U32   : DS_1A1D_RET_mc<"ds_sub_rtn_u32", VGPR_32, "ds_sub_u32">;
533defm DS_RSUB_RTN_U32  : DS_1A1D_RET_mc<"ds_rsub_rtn_u32", VGPR_32, "ds_rsub_u32">;
534defm DS_INC_RTN_U32   : DS_1A1D_RET_mc<"ds_inc_rtn_u32", VGPR_32, "ds_inc_u32">;
535defm DS_DEC_RTN_U32   : DS_1A1D_RET_mc<"ds_dec_rtn_u32", VGPR_32, "ds_dec_u32">;
536defm DS_MIN_RTN_I32   : DS_1A1D_RET_mc<"ds_min_rtn_i32", VGPR_32, "ds_min_i32">;
537defm DS_MAX_RTN_I32   : DS_1A1D_RET_mc<"ds_max_rtn_i32", VGPR_32, "ds_max_i32">;
538defm DS_MIN_RTN_U32   : DS_1A1D_RET_mc<"ds_min_rtn_u32", VGPR_32, "ds_min_u32">;
539defm DS_MAX_RTN_U32   : DS_1A1D_RET_mc<"ds_max_rtn_u32", VGPR_32, "ds_max_u32">;
540defm DS_AND_RTN_B32   : DS_1A1D_RET_mc<"ds_and_rtn_b32", VGPR_32, "ds_and_b32">;
541defm DS_OR_RTN_B32    : DS_1A1D_RET_mc<"ds_or_rtn_b32", VGPR_32, "ds_or_b32">;
542defm DS_XOR_RTN_B32   : DS_1A1D_RET_mc<"ds_xor_rtn_b32", VGPR_32, "ds_xor_b32">;
543defm DS_MSKOR_RTN_B32 : DS_1A2D_RET_mc<"ds_mskor_rtn_b32", VGPR_32, "ds_mskor_b32">;
544defm DS_CMPST_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_b32", VGPR_32, "ds_cmpst_b32">;
545defm DS_CMPST_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_f32", VGPR_32, "ds_cmpst_f32">;
546defm DS_MIN_RTN_F32   : DS_1A1D_RET_mc<"ds_min_rtn_f32", VGPR_32, "ds_min_f32">;
547defm DS_MAX_RTN_F32   : DS_1A1D_RET_mc<"ds_max_rtn_f32", VGPR_32, "ds_max_f32">;
548
549defm DS_WRXCHG_RTN_B32 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b32">;
550defm DS_WRXCHG2_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b32", VReg_64, VGPR_32>;
551defm DS_WRXCHG2ST64_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b32", VReg_64, VGPR_32>;
552
553defm DS_ADD_RTN_U64  : DS_1A1D_RET_mc<"ds_add_rtn_u64", VReg_64, "ds_add_u64">;
554defm DS_SUB_RTN_U64  : DS_1A1D_RET_mc<"ds_sub_rtn_u64", VReg_64, "ds_sub_u64">;
555defm DS_RSUB_RTN_U64  : DS_1A1D_RET_mc<"ds_rsub_rtn_u64", VReg_64, "ds_rsub_u64">;
556defm DS_INC_RTN_U64   : DS_1A1D_RET_mc<"ds_inc_rtn_u64", VReg_64, "ds_inc_u64">;
557defm DS_DEC_RTN_U64   : DS_1A1D_RET_mc<"ds_dec_rtn_u64", VReg_64, "ds_dec_u64">;
558defm DS_MIN_RTN_I64    : DS_1A1D_RET_mc<"ds_min_rtn_i64", VReg_64, "ds_min_i64">;
559defm DS_MAX_RTN_I64    : DS_1A1D_RET_mc<"ds_max_rtn_i64", VReg_64, "ds_max_i64">;
560defm DS_MIN_RTN_U64   : DS_1A1D_RET_mc<"ds_min_rtn_u64", VReg_64, "ds_min_u64">;
561defm DS_MAX_RTN_U64   : DS_1A1D_RET_mc<"ds_max_rtn_u64", VReg_64, "ds_max_u64">;
562defm DS_AND_RTN_B64    : DS_1A1D_RET_mc<"ds_and_rtn_b64", VReg_64, "ds_and_b64">;
563defm DS_OR_RTN_B64     : DS_1A1D_RET_mc<"ds_or_rtn_b64", VReg_64, "ds_or_b64">;
564defm DS_XOR_RTN_B64    : DS_1A1D_RET_mc<"ds_xor_rtn_b64", VReg_64, "ds_xor_b64">;
565defm DS_MSKOR_RTN_B64  : DS_1A2D_RET_mc<"ds_mskor_rtn_b64", VReg_64, "ds_mskor_b64">;
566defm DS_CMPST_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_b64", VReg_64, "ds_cmpst_b64">;
567defm DS_CMPST_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_f64", VReg_64, "ds_cmpst_f64">;
568defm DS_MIN_RTN_F64    : DS_1A1D_RET_mc<"ds_min_rtn_f64", VReg_64, "ds_min_f64">;
569defm DS_MAX_RTN_F64    : DS_1A1D_RET_mc<"ds_max_rtn_f64", VReg_64, "ds_max_f64">;
570
571defm DS_WRXCHG_RTN_B64 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b64", VReg_64>;
572defm DS_WRXCHG2_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b64", VReg_128, VReg_64>;
573defm DS_WRXCHG2ST64_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b64", VReg_128, VReg_64>;
574
575let isConvergent = 1, usesCustomInserter = 1 in {
576def DS_GWS_INIT       : DS_GWS_1D<"ds_gws_init"> {
577  let mayLoad = 0;
578}
579def DS_GWS_SEMA_V     : DS_GWS_0D<"ds_gws_sema_v">;
580def DS_GWS_SEMA_BR    : DS_GWS_1D<"ds_gws_sema_br">;
581def DS_GWS_SEMA_P     : DS_GWS_0D<"ds_gws_sema_p">;
582def DS_GWS_BARRIER    : DS_GWS_1D<"ds_gws_barrier">;
583}
584
585let SubtargetPredicate = HasDsSrc2Insts in {
586def DS_ADD_SRC2_U32   : DS_1A<"ds_add_src2_u32">;
587def DS_SUB_SRC2_U32   : DS_1A<"ds_sub_src2_u32">;
588def DS_RSUB_SRC2_U32  : DS_1A<"ds_rsub_src2_u32">;
589def DS_INC_SRC2_U32   : DS_1A<"ds_inc_src2_u32">;
590def DS_DEC_SRC2_U32   : DS_1A<"ds_dec_src2_u32">;
591def DS_MIN_SRC2_I32   : DS_1A<"ds_min_src2_i32">;
592def DS_MAX_SRC2_I32   : DS_1A<"ds_max_src2_i32">;
593def DS_MIN_SRC2_U32   : DS_1A<"ds_min_src2_u32">;
594def DS_MAX_SRC2_U32   : DS_1A<"ds_max_src2_u32">;
595def DS_AND_SRC2_B32   : DS_1A<"ds_and_src2_b32">;
596def DS_OR_SRC2_B32    : DS_1A<"ds_or_src2_b32">;
597def DS_XOR_SRC2_B32   : DS_1A<"ds_xor_src2_b32">;
598def DS_MIN_SRC2_F32   : DS_1A<"ds_min_src2_f32">;
599def DS_MAX_SRC2_F32   : DS_1A<"ds_max_src2_f32">;
600
601def DS_ADD_SRC2_U64   : DS_1A<"ds_add_src2_u64">;
602def DS_SUB_SRC2_U64   : DS_1A<"ds_sub_src2_u64">;
603def DS_RSUB_SRC2_U64  : DS_1A<"ds_rsub_src2_u64">;
604def DS_INC_SRC2_U64   : DS_1A<"ds_inc_src2_u64">;
605def DS_DEC_SRC2_U64   : DS_1A<"ds_dec_src2_u64">;
606def DS_MIN_SRC2_I64   : DS_1A<"ds_min_src2_i64">;
607def DS_MAX_SRC2_I64   : DS_1A<"ds_max_src2_i64">;
608def DS_MIN_SRC2_U64   : DS_1A<"ds_min_src2_u64">;
609def DS_MAX_SRC2_U64   : DS_1A<"ds_max_src2_u64">;
610def DS_AND_SRC2_B64   : DS_1A<"ds_and_src2_b64">;
611def DS_OR_SRC2_B64    : DS_1A<"ds_or_src2_b64">;
612def DS_XOR_SRC2_B64   : DS_1A<"ds_xor_src2_b64">;
613def DS_MIN_SRC2_F64   : DS_1A<"ds_min_src2_f64">;
614def DS_MAX_SRC2_F64   : DS_1A<"ds_max_src2_f64">;
615
616def DS_WRITE_SRC2_B32 : DS_1A<"ds_write_src2_b32">;
617def DS_WRITE_SRC2_B64 : DS_1A<"ds_write_src2_b64">;
618} // End SubtargetPredicate = HasDsSrc2Insts
619
620let Uses = [EXEC], mayLoad = 0, mayStore = 0, isConvergent = 1 in {
621def DS_SWIZZLE_B32 : DS_1A_RET <"ds_swizzle_b32", VGPR_32, 0, SwizzleImm>;
622}
623
624let mayStore = 0 in {
625defm DS_READ_I8      : DS_1A_RET_mc<"ds_read_i8">;
626defm DS_READ_U8      : DS_1A_RET_mc<"ds_read_u8">;
627defm DS_READ_I16     : DS_1A_RET_mc<"ds_read_i16">;
628defm DS_READ_U16     : DS_1A_RET_mc<"ds_read_u16">;
629defm DS_READ_B32     : DS_1A_RET_mc<"ds_read_b32">;
630defm DS_READ_B64     : DS_1A_RET_mc<"ds_read_b64", VReg_64>;
631
632defm DS_READ2_B32    : DS_1A_Off8_RET_mc<"ds_read2_b32", VReg_64>;
633defm DS_READ2ST64_B32: DS_1A_Off8_RET_mc<"ds_read2st64_b32", VReg_64>;
634
635defm DS_READ2_B64    : DS_1A_Off8_RET_mc<"ds_read2_b64", VReg_128>;
636defm DS_READ2ST64_B64: DS_1A_Off8_RET_mc<"ds_read2st64_b64", VReg_128>;
637
638let has_m0_read = 0 in {
639let SubtargetPredicate = HasD16LoadStore in {
640def DS_READ_U8_D16     : DS_1A_RET_Tied<"ds_read_u8_d16">;
641def DS_READ_U8_D16_HI  : DS_1A_RET_Tied<"ds_read_u8_d16_hi">;
642def DS_READ_I8_D16     : DS_1A_RET_Tied<"ds_read_i8_d16">;
643def DS_READ_I8_D16_HI  : DS_1A_RET_Tied<"ds_read_i8_d16_hi">;
644def DS_READ_U16_D16    : DS_1A_RET_Tied<"ds_read_u16_d16">;
645def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">;
646}
647} // End has_m0_read = 0
648
649let SubtargetPredicate = HasDSAddTid in {
650def DS_READ_ADDTID_B32 : DS_0A_RET<"ds_read_addtid_b32">;
651}
652
653} // End mayStore = 0
654
655def DS_CONSUME       : DS_0A_RET<"ds_consume">;
656def DS_APPEND        : DS_0A_RET<"ds_append">;
657
658let SubtargetPredicate = isNotGFX90APlus in
659def DS_ORDERED_COUNT : DS_1A_RET_GDS<"ds_ordered_count">;
660
661//===----------------------------------------------------------------------===//
662// Instruction definitions for CI and newer.
663//===----------------------------------------------------------------------===//
664
665let SubtargetPredicate = isGFX7Plus in {
666
667defm DS_WRAP_RTN_B32 : DS_1A2D_RET_mc<"ds_wrap_rtn_b32", VGPR_32>;
668defm DS_CONDXCHG32_RTN_B64 : DS_1A1D_RET_mc<"ds_condxchg32_rtn_b64", VReg_64>;
669
670let isConvergent = 1, usesCustomInserter = 1 in {
671def DS_GWS_SEMA_RELEASE_ALL : DS_GWS_0D<"ds_gws_sema_release_all">;
672}
673
674let mayStore = 0 in {
675defm DS_READ_B96 : DS_1A_RET_mc<"ds_read_b96", VReg_96>;
676defm DS_READ_B128: DS_1A_RET_mc<"ds_read_b128", VReg_128>;
677} // End mayStore = 0
678
679let mayLoad = 0 in {
680defm DS_WRITE_B96 : DS_1A1D_NORET_mc<"ds_write_b96", VReg_96>;
681defm DS_WRITE_B128 : DS_1A1D_NORET_mc<"ds_write_b128", VReg_128>;
682} // End mayLoad = 0
683
684def DS_NOP : DS_VOID<"ds_nop">;
685
686} // let SubtargetPredicate = isGFX7Plus
687
688//===----------------------------------------------------------------------===//
689// Instruction definitions for VI and newer.
690//===----------------------------------------------------------------------===//
691
692let SubtargetPredicate = isGFX8Plus in {
693
694let Uses = [EXEC] in {
695def DS_PERMUTE_B32  : DS_1A1D_PERMUTE <"ds_permute_b32",
696                                       int_amdgcn_ds_permute>;
697def DS_BPERMUTE_B32 : DS_1A1D_PERMUTE <"ds_bpermute_b32",
698                                       int_amdgcn_ds_bpermute>;
699}
700
701} // let SubtargetPredicate = isGFX8Plus
702
703let SubtargetPredicate = HasLDSFPAtomicAdd, OtherPredicates = [HasDsSrc2Insts] in {
704def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;
705}
706
707
708//===----------------------------------------------------------------------===//
709// Instruction definitions for GFX11 and newer.
710//===----------------------------------------------------------------------===//
711
712let SubtargetPredicate = isGFX11Plus in {
713
714def DS_ADD_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_add_gs_reg_rtn", VReg_64, VGPR_32>;
715def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VReg_64, VGPR_32>;
716
717} // let SubtargetPredicate = isGFX11Plus
718
719//===----------------------------------------------------------------------===//
720// DS Patterns
721//===----------------------------------------------------------------------===//
722
723def : GCNPat <
724  (int_amdgcn_ds_swizzle i32:$src, timm:$offset16),
725  (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0))
726>;
727
728class DSReadPat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
729  (vt (frag (DS1Addr1Offset i32:$ptr, i16:$offset))),
730  (inst $ptr, offset:$offset, (i1 gds))
731>;
732
733multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
734
735  let OtherPredicates = [LDSRequiresM0Init] in {
736    def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
737  }
738
739  let OtherPredicates = [NotLDSRequiresM0Init] in {
740    def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
741  }
742}
743
744class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <
745  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$in),
746  (inst $ptr, offset:$offset, (i1 0), $in)
747>;
748
749defm : DSReadPat_mc <DS_READ_I8, i32, "sextloadi8_local">;
750defm : DSReadPat_mc <DS_READ_I8,  i16, "sextloadi8_local">;
751defm : DSReadPat_mc <DS_READ_U8,  i32, "extloadi8_local">;
752defm : DSReadPat_mc <DS_READ_U8,  i32, "zextloadi8_local">;
753defm : DSReadPat_mc <DS_READ_U8,  i16, "extloadi8_local">;
754defm : DSReadPat_mc <DS_READ_U8,  i16, "zextloadi8_local">;
755defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
756defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
757defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
758defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
759defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
760
761foreach vt = Reg32Types.types in {
762defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
763}
764
765defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_8_local">;
766defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_8_local">;
767defm : DSReadPat_mc <DS_READ_U16, i16, "atomic_load_16_local">;
768defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_16_local">;
769defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
770defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
771
772let OtherPredicates = [D16PreservesUnusedBits] in {
773def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
774def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
775def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
776def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;
777def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2i16>;
778def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2f16>;
779
780def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2i16>;
781def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2f16>;
782def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2i16>;
783def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2f16>;
784def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2i16>;
785def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2f16>;
786}
787
788class DSWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
789  (frag vt:$value, (DS1Addr1Offset i32:$ptr, i16:$offset)),
790  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
791>;
792
793multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
794  let OtherPredicates = [LDSRequiresM0Init] in {
795    def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
796  }
797
798  let OtherPredicates = [NotLDSRequiresM0Init] in {
799    def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
800  }
801}
802
803// Irritatingly, atomic_store reverses the order of operands from a
804// normal store.
805class DSAtomicWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
806  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
807  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 0))
808>;
809
810multiclass DSAtomicWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
811  let OtherPredicates = [LDSRequiresM0Init] in {
812    def : DSAtomicWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
813  }
814
815  let OtherPredicates = [NotLDSRequiresM0Init] in {
816    def : DSAtomicWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
817  }
818}
819
820defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
821defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
822defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
823defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
824
825foreach vt = Reg32Types.types in {
826defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
827}
828
829defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
830defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
831defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
832defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
833defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
834defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
835
836let OtherPredicates = [HasD16LoadStore] in {
837def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
838def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;
839}
840
841class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
842  (vt:$value (frag (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
843  (inst $ptr, $offset0, $offset1, (i1 0))
844>;
845
846class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
847  (frag vt:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
848  (inst $ptr, (i32 (EXTRACT_SUBREG VReg_64:$value, sub0)),
849              (i32 (EXTRACT_SUBREG VReg_64:$value, sub1)), $offset0, $offset1,
850              (i1 0))
851>;
852
853class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
854  (vt:$value (frag (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
855  (inst $ptr, $offset0, $offset1, (i1 0))
856>;
857
858class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
859  (frag vt:$value, (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
860  (inst $ptr, (i64 (EXTRACT_SUBREG VReg_128:$value, sub0_sub1)),
861              (i64 (EXTRACT_SUBREG VReg_128:$value, sub2_sub3)), $offset0, $offset1,
862              (i1 0))
863>;
864
865multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {
866  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
867    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32, vt, load_local_m0>;
868    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_local_m0>;
869  }
870
871  let OtherPredicates = [NotLDSRequiresM0Init] in {
872    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, vt, load_local>;
873    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, vt, store_local>;
874  }
875}
876
877multiclass DS128Bit8ByteAlignedPat_mc<ValueType vt> {
878  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
879    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64, vt, load_local_m0>;
880    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_local_m0>;
881  }
882
883  let OtherPredicates = [NotLDSRequiresM0Init] in {
884    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64_gfx9, vt, load_local>;
885    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64_gfx9, vt, store_local>;
886  }
887}
888
889// v2i32 loads are split into i32 loads on SI during lowering, due to a bug
890// related to bounds checking.
891foreach vt = VReg_64.RegTypes in {
892defm : DS64Bit4ByteAlignedPat_mc<vt>;
893}
894
895foreach vt = VReg_128.RegTypes in {
896defm : DS128Bit8ByteAlignedPat_mc<vt>;
897}
898
899// Prefer ds_read over ds_read2 and ds_write over ds_write2, all other things
900// being equal, because it has a larger immediate offset range.
901let AddedComplexity = 100 in {
902
903foreach vt = VReg_64.RegTypes in {
904defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
905defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
906}
907
908let SubtargetPredicate = isGFX7Plus in {
909
910foreach vt = VReg_96.RegTypes in {
911defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
912defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
913}
914
915foreach vt = VReg_128.RegTypes in {
916defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
917defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
918}
919
920let SubtargetPredicate = HasUnalignedAccessMode in {
921
922// Select 64 bit loads and stores aligned less than 4 as a single ds_read_b64/
923// ds_write_b64 instruction as this is faster than ds_read2_b32/ds_write2_b32
924// which would be used otherwise. In this case a b32 access would still be
925// misaligned, but we will have 2 of them.
926foreach vt = VReg_64.RegTypes in {
927defm : DSReadPat_mc <DS_READ_B64, vt, "load_align_less_than_4_local">;
928defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align_less_than_4_local">;
929}
930
931// Selection will split most of the unaligned 3 dword accesses due to performance
932// reasons when beneficial. Keep these two patterns for the rest of the cases.
933foreach vt = VReg_96.RegTypes in {
934defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
935defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
936}
937
938// Select 128 bit loads and stores aligned less than 4 as a single ds_read_b128/
939// ds_write_b128 instruction as this is faster than ds_read2_b64/ds_write2_b64
940// which would be used otherwise. In this case a b64 access would still be
941// misaligned, but we will have 2 of them.
942foreach vt = VReg_128.RegTypes in {
943defm : DSReadPat_mc <DS_READ_B128, vt, "load_align_less_than_4_local">;
944defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align_less_than_4_local">;
945}
946
947} // End SubtargetPredicate = HasUnalignedAccessMode
948
949} // End SubtargetPredicate = isGFX7Plus
950
951} // End AddedComplexity = 100
952
953class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, int complexity = 0,
954  bit gds=0> : GCNPat <(frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
955  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))> {
956  let AddedComplexity = complexity;
957}
958
959multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
960  let OtherPredicates = [LDSRequiresM0Init] in {
961    def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
962  }
963
964  let OtherPredicates = [NotLDSRequiresM0Init] in {
965    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
966                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
967  }
968
969  def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
970                       /* complexity */ 0, /* gds */ 1>;
971}
972
973multiclass DSAtomicRetNoRetPat_mc<DS_Pseudo inst, DS_Pseudo noRetInst,
974                                  ValueType vt, string frag> {
975  let OtherPredicates = [LDSRequiresM0Init] in {
976    def : DSAtomicRetPat<inst, vt,
977                         !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
978    def : DSAtomicRetPat<noRetInst, vt,
979                         !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size), /* complexity */ 1>;
980  }
981
982  let OtherPredicates = [NotLDSRequiresM0Init] in {
983    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
984                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
985    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
986                         !cast<PatFrag>(frag#"_local_noret_"#vt.Size), /* complexity */ 1>;
987  }
988
989  def : DSAtomicRetPat<inst, vt,
990                       !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
991                       /* complexity */ 0, /* gds */ 1>;
992  def : DSAtomicRetPat<noRetInst, vt,
993                       !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
994                       /* complexity */ 1, /* gds */ 1>;
995}
996
997
998
999let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1000// Caution, the order of src and cmp is the *opposite* of the BUFFER_ATOMIC_CMPSWAP opcode.
1001class DSAtomicCmpXChgSwapped<DS_Pseudo inst, ValueType vt, PatFrag frag,
1002  int complexity = 0, bit gds=0> : GCNPat<
1003  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap),
1004  (inst $ptr, getVregSrcForVT<vt>.ret:$cmp, getVregSrcForVT<vt>.ret:$swap, offset:$offset, (i1 gds))> {
1005  let AddedComplexity = complexity;
1006}
1007
1008multiclass DSAtomicCmpXChgSwapped_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt,
1009                                     string frag> {
1010  let OtherPredicates = [LDSRequiresM0Init] in {
1011    def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
1012    def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size),
1013                                 /* complexity */ 1>;
1014  }
1015
1016  let OtherPredicates = [NotLDSRequiresM0Init] in {
1017    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1018                                 !cast<PatFrag>(frag#"_local_"#vt.Size)>;
1019    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1020                                 !cast<PatFrag>(frag#"_local_noret_"#vt.Size),
1021                                 /* complexity */ 1>;
1022  }
1023
1024  def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1025                               /* complexity */ 0, /* gds */ 1>;
1026  def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1027                               /* complexity */ 1, /* gds */ 1>;
1028}
1029} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1030
1031let SubtargetPredicate = isGFX11Plus in {
1032// The order of src and cmp agrees with the BUFFER_ATOMIC_CMPSWAP opcode.
1033class DSAtomicCmpXChg<DS_Pseudo inst, ValueType vt, PatFrag frag,
1034  int complexity = 0, bit gds=0> : GCNPat<
1035  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap),
1036  (inst $ptr, getVregSrcForVT<vt>.ret:$swap, getVregSrcForVT<vt>.ret:$cmp, offset:$offset, (i1 gds))> {
1037  let AddedComplexity = complexity;
1038}
1039
1040multiclass DSAtomicCmpXChg_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt, string frag> {
1041
1042  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1043                        !cast<PatFrag>(frag#"_local_"#vt.Size)>;
1044  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1045                        !cast<PatFrag>(frag#"_local_noret_"#vt.Size), /* complexity */ 1>;
1046
1047  def : DSAtomicCmpXChg<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1048                        /* complexity */ 0, /* gds */ 1>;
1049  def : DSAtomicCmpXChg<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1050                        /* complexity */ 1, /* gds */ 1>;
1051}
1052} // End SubtargetPredicate = isGFX11Plus
1053
1054// 32-bit atomics.
1055defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B32, i32, "atomic_swap">;
1056defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U32, DS_ADD_U32, i32, "atomic_load_add">;
1057defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U32, DS_SUB_U32, i32, "atomic_load_sub">;
1058defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_inc">;
1059defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_dec">;
1060defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B32, DS_AND_B32, i32, "atomic_load_and">;
1061defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B32, DS_OR_B32, i32, "atomic_load_or">;
1062defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B32, DS_XOR_B32, i32, "atomic_load_xor">;
1063defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I32, DS_MIN_I32, i32, "atomic_load_min">;
1064defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I32, DS_MAX_I32, i32, "atomic_load_max">;
1065defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U32, DS_MIN_U32, i32, "atomic_load_umin">;
1066defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U32, DS_MAX_U32, i32, "atomic_load_umax">;
1067defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F32, DS_MIN_F32, f32, "atomic_load_fmin">;
1068defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F32, DS_MAX_F32, f32, "atomic_load_fmax">;
1069
1070let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1071defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B32, DS_CMPST_B32, i32, "atomic_cmp_swap">;
1072}
1073
1074let SubtargetPredicate = isGFX11Plus in {
1075defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B32, DS_CMPSTORE_B32, i32, "atomic_cmp_swap">;
1076}
1077
1078let SubtargetPredicate = HasLDSFPAtomicAdd in {
1079defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_F32, DS_ADD_F32, f32, "atomic_load_fadd">;
1080}
1081
1082// 64-bit atomics.
1083defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B64, i64, "atomic_swap">;
1084defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U64, DS_ADD_U64, i64, "atomic_load_add">;
1085defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U64, DS_SUB_U64, i64, "atomic_load_sub">;
1086defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_inc">;
1087defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_dec">;
1088defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B64, DS_AND_B64, i64, "atomic_load_and">;
1089defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B64, DS_OR_B64, i64, "atomic_load_or">;
1090defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B64, DS_XOR_B64, i64, "atomic_load_xor">;
1091defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I64, DS_MIN_I64, i64, "atomic_load_min">;
1092defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I64, DS_MAX_I64, i64, "atomic_load_max">;
1093defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U64, DS_MIN_U64, i64, "atomic_load_umin">;
1094defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U64, DS_MAX_U64, i64, "atomic_load_umax">;
1095defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F64, DS_MIN_F64, f64, "atomic_load_fmin">;
1096defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F64, DS_MAX_F64, f64, "atomic_load_fmax">;
1097
1098let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1099defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B64, DS_CMPST_B64, i64, "atomic_cmp_swap">;
1100} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1101
1102let SubtargetPredicate = isGFX11Plus in {
1103defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B64, DS_CMPSTORE_B64, i64, "atomic_cmp_swap">;
1104} // End SubtargetPredicate = isGFX11Plus
1105
1106let SubtargetPredicate = isGFX90APlus in {
1107def : DSAtomicRetPat<DS_ADD_RTN_F64, f64, atomic_load_fadd_local_64>;
1108let AddedComplexity = 1 in
1109def : DSAtomicRetPat<DS_ADD_F64, f64, atomic_load_fadd_local_noret_64>;
1110}
1111
1112let SubtargetPredicate = isGFX940Plus in {
1113def : DSAtomicRetPat<DS_PK_ADD_RTN_F16, v2f16, atomic_load_fadd_v2f16_local_32>;
1114let AddedComplexity = 1 in
1115def : DSAtomicRetPat<DS_PK_ADD_F16, v2f16, atomic_load_fadd_v2f16_local_noret_32>;
1116def : GCNPat <
1117  (v2i16 (int_amdgcn_ds_fadd_v2bf16 i32:$ptr, v2i16:$src)),
1118  (DS_PK_ADD_RTN_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1119>;
1120let AddedComplexity = 1 in
1121def : GCNPat <
1122  (v2i16 (int_amdgcn_ds_fadd_v2bf16_noret i32:$ptr, v2i16:$src)),
1123  (DS_PK_ADD_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1124>;
1125}
1126
1127def : Pat <
1128  (SIds_ordered_count i32:$value, i16:$offset),
1129  (DS_ORDERED_COUNT $value, (as_i16imm $offset))
1130>;
1131
1132def : GCNPat <
1133  (i64 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),
1134  (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))
1135>;
1136
1137def : GCNPat <
1138  (i32 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),
1139  (EXTRACT_SUBREG
1140    (i64 (COPY_TO_REGCLASS
1141      (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),
1142      VReg_64)),
1143    sub0)
1144>;
1145
1146def : GCNPat <
1147  (i64 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),
1148  (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))
1149>;
1150
1151def : GCNPat <
1152  (i32 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),
1153  (EXTRACT_SUBREG
1154    (i64 (COPY_TO_REGCLASS
1155      (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),
1156      VReg_64)),
1157    sub0)
1158>;
1159
1160//===----------------------------------------------------------------------===//
1161// Target-specific instruction encodings.
1162//===----------------------------------------------------------------------===//
1163
1164//===----------------------------------------------------------------------===//
1165// Base ENC_DS for GFX6, GFX7, GFX10, GFX11.
1166//===----------------------------------------------------------------------===//
1167
1168class Base_DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op, DS_Pseudo ps, int ef, string opName = ps.Mnemonic> :
1169    DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> {
1170
1171  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1172  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1173  let Inst{17}    = !if(ps.has_gds, gds, ps.gdsValue);
1174  let Inst{25-18} = op;
1175  let Inst{31-26} = 0x36;
1176  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1177  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1178  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1179  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1180}
1181
1182//===----------------------------------------------------------------------===//
1183// GFX11.
1184//===----------------------------------------------------------------------===//
1185
1186let AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11" in {
1187  multiclass DS_Real_gfx11<bits<8> op>  {
1188    def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1189                                              SIEncodingFamily.GFX11>;
1190  }
1191
1192  multiclass DS_Real_Renamed_gfx11<bits<8> op, DS_Pseudo backing_pseudo, string real_name> {
1193     def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, backing_pseudo, SIEncodingFamily.GFX11, real_name>,
1194               MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX11Plus]>;
1195  }
1196} // End AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11"
1197
1198defm DS_STORE_B32                        : DS_Real_Renamed_gfx11<0x00d, DS_WRITE_B32, "ds_store_b32">;
1199defm DS_STORE_2ADDR_B32                  : DS_Real_Renamed_gfx11<0x00e, DS_WRITE2_B32, "ds_store_2addr_b32">;
1200defm DS_STORE_2ADDR_STRIDE64_B32         : DS_Real_Renamed_gfx11<0x00f, DS_WRITE2ST64_B32, "ds_store_2addr_stride64_b32">;
1201defm DS_STORE_B8                         : DS_Real_Renamed_gfx11<0x01e, DS_WRITE_B8, "ds_store_b8">;
1202defm DS_STORE_B16                        : DS_Real_Renamed_gfx11<0x01f, DS_WRITE_B16, "ds_store_b16">;
1203defm DS_STOREXCHG_RTN_B32                : DS_Real_Renamed_gfx11<0x02d, DS_WRXCHG_RTN_B32, "ds_storexchg_rtn_b32">;
1204defm DS_STOREXCHG_2ADDR_RTN_B32          : DS_Real_Renamed_gfx11<0x02e, DS_WRXCHG2_RTN_B32, "ds_storexchg_2addr_rtn_b32">;
1205defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32 : DS_Real_Renamed_gfx11<0x02f, DS_WRXCHG2ST64_RTN_B32, "ds_storexchg_2addr_stride64_rtn_b32">;
1206defm DS_LOAD_B32                         : DS_Real_Renamed_gfx11<0x036, DS_READ_B32, "ds_load_b32">;
1207defm DS_LOAD_2ADDR_B32                   : DS_Real_Renamed_gfx11<0x037, DS_READ2_B32, "ds_load_2addr_b32">;
1208defm DS_LOAD_2ADDR_STRIDE64_B32          : DS_Real_Renamed_gfx11<0x038, DS_READ2ST64_B32, "ds_load_2addr_stride64_b32">;
1209defm DS_LOAD_I8                          : DS_Real_Renamed_gfx11<0x039, DS_READ_I8, "ds_load_i8">;
1210defm DS_LOAD_U8                          : DS_Real_Renamed_gfx11<0x03a, DS_READ_U8, "ds_load_u8">;
1211defm DS_LOAD_I16                         : DS_Real_Renamed_gfx11<0x03b, DS_READ_I16, "ds_load_i16">;
1212defm DS_LOAD_U16                         : DS_Real_Renamed_gfx11<0x03c, DS_READ_U16, "ds_load_u16">;
1213defm DS_STORE_B64                        : DS_Real_Renamed_gfx11<0x04d, DS_WRITE_B64, "ds_store_b64">;
1214defm DS_STORE_2ADDR_B64                  : DS_Real_Renamed_gfx11<0x04e, DS_WRITE2_B64, "ds_store_2addr_b64">;
1215defm DS_STORE_2ADDR_STRIDE64_B64         : DS_Real_Renamed_gfx11<0x04f, DS_WRITE2ST64_B64, "ds_store_2addr_stride64_b64">;
1216defm DS_STOREXCHG_RTN_B64                : DS_Real_Renamed_gfx11<0x06d, DS_WRXCHG_RTN_B64, "ds_storexchg_rtn_b64">;
1217defm DS_STOREXCHG_2ADDR_RTN_B64          : DS_Real_Renamed_gfx11<0x06e, DS_WRXCHG2_RTN_B64, "ds_storexchg_2addr_rtn_b64">;
1218defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64 : DS_Real_Renamed_gfx11<0x06f, DS_WRXCHG2ST64_RTN_B64, "ds_storexchg_2addr_stride64_rtn_b64">;
1219defm DS_LOAD_B64                         : DS_Real_Renamed_gfx11<0x076, DS_READ_B64, "ds_load_b64">;
1220defm DS_LOAD_2ADDR_B64                   : DS_Real_Renamed_gfx11<0x077, DS_READ2_B64, "ds_load_2addr_b64">;
1221defm DS_LOAD_2ADDR_STRIDE64_B64          : DS_Real_Renamed_gfx11<0x078, DS_READ2ST64_B64, "ds_load_2addr_stride64_b64">;
1222defm DS_STORE_B8_D16_HI                  : DS_Real_Renamed_gfx11<0x0a0, DS_WRITE_B8_D16_HI, "ds_store_b8_d16_hi">;
1223defm DS_STORE_B16_D16_HI                 : DS_Real_Renamed_gfx11<0x0a1, DS_WRITE_B16_D16_HI, "ds_store_b16_d16_hi">;
1224defm DS_LOAD_U8_D16                      : DS_Real_Renamed_gfx11<0x0a2, DS_READ_U8_D16, "ds_load_u8_d16">;
1225defm DS_LOAD_U8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a3, DS_READ_U8_D16_HI, "ds_load_u8_d16_hi">;
1226defm DS_LOAD_I8_D16                      : DS_Real_Renamed_gfx11<0x0a4, DS_READ_I8_D16, "ds_load_i8_d16">;
1227defm DS_LOAD_I8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a5, DS_READ_I8_D16_HI, "ds_load_i8_d16_hi">;
1228defm DS_LOAD_U16_D16                     : DS_Real_Renamed_gfx11<0x0a6, DS_READ_U16_D16, "ds_load_u16_d16">;
1229defm DS_LOAD_U16_D16_HI                  : DS_Real_Renamed_gfx11<0x0a7, DS_READ_U16_D16_HI, "ds_load_u16_d16_hi">;
1230defm DS_STORE_ADDTID_B32                 : DS_Real_Renamed_gfx11<0x0b0, DS_WRITE_ADDTID_B32, "ds_store_addtid_b32">;
1231defm DS_LOAD_ADDTID_B32                  : DS_Real_Renamed_gfx11<0x0b1, DS_READ_ADDTID_B32, "ds_load_addtid_b32">;
1232defm DS_STORE_B96                        : DS_Real_Renamed_gfx11<0x0de, DS_WRITE_B96, "ds_store_b96">;
1233defm DS_STORE_B128                       : DS_Real_Renamed_gfx11<0x0df, DS_WRITE_B128, "ds_store_b128">;
1234defm DS_LOAD_B96                         : DS_Real_Renamed_gfx11<0x0fe, DS_READ_B96, "ds_load_b96">;
1235defm DS_LOAD_B128                        : DS_Real_Renamed_gfx11<0x0ff, DS_READ_B128, "ds_load_b128">;
1236
1237// DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped
1238// comparing to pre-GFX11.
1239// Note: the mnemonic alias is not generated to avoid a potential ambiguity due to the semantics change.
1240
1241defm DS_CMPSTORE_B32                     : DS_Real_gfx11<0x010>;
1242defm DS_CMPSTORE_F32                     : DS_Real_gfx11<0x011>;
1243defm DS_CMPSTORE_RTN_B32                 : DS_Real_gfx11<0x030>;
1244defm DS_CMPSTORE_RTN_F32                 : DS_Real_gfx11<0x031>;
1245defm DS_CMPSTORE_B64                     : DS_Real_gfx11<0x050>;
1246defm DS_CMPSTORE_F64                     : DS_Real_gfx11<0x051>;
1247defm DS_CMPSTORE_RTN_B64                 : DS_Real_gfx11<0x070>;
1248defm DS_CMPSTORE_RTN_F64                 : DS_Real_gfx11<0x071>;
1249
1250defm DS_ADD_RTN_F32                      : DS_Real_gfx11<0x079>;
1251defm DS_ADD_GS_REG_RTN                   : DS_Real_gfx11<0x07a>;
1252defm DS_SUB_GS_REG_RTN                   : DS_Real_gfx11<0x07b>;
1253
1254//===----------------------------------------------------------------------===//
1255// GFX10.
1256//===----------------------------------------------------------------------===//
1257
1258let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
1259  multiclass DS_Real_gfx10<bits<8> op>  {
1260    def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1261                                              SIEncodingFamily.GFX10>;
1262  }
1263} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
1264
1265defm DS_ADD_RTN_F32      : DS_Real_gfx10<0x055>;
1266defm DS_WRITE_B8_D16_HI  : DS_Real_gfx10<0x0a0>;
1267defm DS_WRITE_B16_D16_HI : DS_Real_gfx10<0x0a1>;
1268defm DS_READ_U8_D16      : DS_Real_gfx10<0x0a2>;
1269defm DS_READ_U8_D16_HI   : DS_Real_gfx10<0x0a3>;
1270defm DS_READ_I8_D16      : DS_Real_gfx10<0x0a4>;
1271defm DS_READ_I8_D16_HI   : DS_Real_gfx10<0x0a5>;
1272defm DS_READ_U16_D16     : DS_Real_gfx10<0x0a6>;
1273defm DS_READ_U16_D16_HI  : DS_Real_gfx10<0x0a7>;
1274defm DS_WRITE_ADDTID_B32 : DS_Real_gfx10<0x0b0>;
1275defm DS_READ_ADDTID_B32  : DS_Real_gfx10<0x0b1>;
1276
1277//===----------------------------------------------------------------------===//
1278// GFX10, GFX11.
1279//===----------------------------------------------------------------------===//
1280
1281multiclass DS_Real_gfx10_gfx11<bits<8> op> :
1282  DS_Real_gfx10<op>, DS_Real_gfx11<op>;
1283
1284defm DS_ADD_F32          : DS_Real_gfx10_gfx11<0x015>;
1285defm DS_ADD_SRC2_F32     : DS_Real_gfx10<0x095>;
1286defm DS_PERMUTE_B32      : DS_Real_gfx10_gfx11<0x0b2>;
1287defm DS_BPERMUTE_B32     : DS_Real_gfx10_gfx11<0x0b3>;
1288
1289//===----------------------------------------------------------------------===//
1290// GFX7, GFX10, GFX11.
1291//===----------------------------------------------------------------------===//
1292
1293let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1294  multiclass DS_Real_gfx7<bits<8> op> {
1295    def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1296                                             SIEncodingFamily.SI>;
1297  }
1298} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1299
1300multiclass DS_Real_gfx7_gfx10_gfx11<bits<8> op> :
1301  DS_Real_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1302
1303multiclass DS_Real_gfx7_gfx10<bits<8> op> :
1304  DS_Real_gfx7<op>, DS_Real_gfx10<op>;
1305
1306// FIXME-GFX7: Add tests when upstreaming this part.
1307defm DS_GWS_SEMA_RELEASE_ALL : DS_Real_gfx7_gfx10_gfx11<0x018>;
1308defm DS_WRAP_RTN_B32         : DS_Real_gfx7_gfx10_gfx11<0x034>;
1309defm DS_CONDXCHG32_RTN_B64   : DS_Real_gfx7_gfx10_gfx11<0x07e>;
1310defm DS_WRITE_B96            : DS_Real_gfx7_gfx10<0x0de>;
1311defm DS_WRITE_B128           : DS_Real_gfx7_gfx10<0x0df>;
1312defm DS_READ_B96             : DS_Real_gfx7_gfx10<0x0fe>;
1313defm DS_READ_B128            : DS_Real_gfx7_gfx10<0x0ff>;
1314
1315//===----------------------------------------------------------------------===//
1316// GFX6, GFX7, GFX10, GFX11.
1317//===----------------------------------------------------------------------===//
1318
1319let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1320  multiclass DS_Real_gfx6_gfx7<bits<8> op> {
1321    def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1322                                                  SIEncodingFamily.SI>;
1323  }
1324} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1325
1326multiclass DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op> :
1327  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1328
1329multiclass DS_Real_gfx6_gfx7_gfx10<bits<8> op> :
1330  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10<op>;
1331
1332defm DS_ADD_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x000>;
1333defm DS_SUB_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x001>;
1334defm DS_RSUB_U32            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x002>;
1335defm DS_INC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x003>;
1336defm DS_DEC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x004>;
1337defm DS_MIN_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x005>;
1338defm DS_MAX_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x006>;
1339defm DS_MIN_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x007>;
1340defm DS_MAX_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x008>;
1341defm DS_AND_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x009>;
1342defm DS_OR_B32              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00a>;
1343defm DS_XOR_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00b>;
1344defm DS_MSKOR_B32           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00c>;
1345
1346defm DS_WRITE_B32           : DS_Real_gfx6_gfx7_gfx10<0x00d>;
1347defm DS_WRITE2_B32          : DS_Real_gfx6_gfx7_gfx10<0x00e>;
1348defm DS_WRITE2ST64_B32      : DS_Real_gfx6_gfx7_gfx10<0x00f>;
1349defm DS_CMPST_B32           : DS_Real_gfx6_gfx7_gfx10<0x010>;
1350defm DS_CMPST_F32           : DS_Real_gfx6_gfx7_gfx10<0x011>;
1351
1352defm DS_MIN_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x012>;
1353defm DS_MAX_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x013>;
1354defm DS_NOP                 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x014>;
1355defm DS_GWS_INIT            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x019>;
1356defm DS_GWS_SEMA_V          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01a>;
1357defm DS_GWS_SEMA_BR         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01b>;
1358defm DS_GWS_SEMA_P          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01c>;
1359defm DS_GWS_BARRIER         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01d>;
1360
1361defm DS_WRITE_B8            : DS_Real_gfx6_gfx7_gfx10<0x01e>;
1362defm DS_WRITE_B16           : DS_Real_gfx6_gfx7_gfx10<0x01f>;
1363
1364defm DS_ADD_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x020>;
1365defm DS_SUB_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x021>;
1366defm DS_RSUB_RTN_U32        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x022>;
1367defm DS_INC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x023>;
1368defm DS_DEC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x024>;
1369defm DS_MIN_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x025>;
1370defm DS_MAX_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x026>;
1371defm DS_MIN_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x027>;
1372defm DS_MAX_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x028>;
1373defm DS_AND_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x029>;
1374defm DS_OR_RTN_B32          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02a>;
1375defm DS_XOR_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02b>;
1376defm DS_MSKOR_RTN_B32       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02c>;
1377
1378defm DS_WRXCHG_RTN_B32      : DS_Real_gfx6_gfx7_gfx10<0x02d>;
1379defm DS_WRXCHG2_RTN_B32     : DS_Real_gfx6_gfx7_gfx10<0x02e>;
1380defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02f>;
1381defm DS_CMPST_RTN_B32       : DS_Real_gfx6_gfx7_gfx10<0x030>;
1382defm DS_CMPST_RTN_F32       : DS_Real_gfx6_gfx7_gfx10<0x031>;
1383
1384defm DS_MIN_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x032>;
1385defm DS_MAX_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x033>;
1386defm DS_SWIZZLE_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x035>;
1387
1388defm DS_READ_B32            : DS_Real_gfx6_gfx7_gfx10<0x036>;
1389defm DS_READ2_B32           : DS_Real_gfx6_gfx7_gfx10<0x037>;
1390defm DS_READ2ST64_B32       : DS_Real_gfx6_gfx7_gfx10<0x038>;
1391defm DS_READ_I8             : DS_Real_gfx6_gfx7_gfx10<0x039>;
1392defm DS_READ_U8             : DS_Real_gfx6_gfx7_gfx10<0x03a>;
1393defm DS_READ_I16            : DS_Real_gfx6_gfx7_gfx10<0x03b>;
1394defm DS_READ_U16            : DS_Real_gfx6_gfx7_gfx10<0x03c>;
1395
1396defm DS_CONSUME             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03d>;
1397defm DS_APPEND              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03e>;
1398defm DS_ORDERED_COUNT       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03f>;
1399defm DS_ADD_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x040>;
1400defm DS_SUB_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x041>;
1401defm DS_RSUB_U64            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x042>;
1402defm DS_INC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x043>;
1403defm DS_DEC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x044>;
1404defm DS_MIN_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x045>;
1405defm DS_MAX_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x046>;
1406defm DS_MIN_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x047>;
1407defm DS_MAX_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x048>;
1408defm DS_AND_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x049>;
1409defm DS_OR_B64              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04a>;
1410defm DS_XOR_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04b>;
1411defm DS_MSKOR_B64           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04c>;
1412
1413defm DS_WRITE_B64           : DS_Real_gfx6_gfx7_gfx10<0x04d>;
1414defm DS_WRITE2_B64          : DS_Real_gfx6_gfx7_gfx10<0x04e>;
1415defm DS_WRITE2ST64_B64      : DS_Real_gfx6_gfx7_gfx10<0x04f>;
1416defm DS_CMPST_B64           : DS_Real_gfx6_gfx7_gfx10<0x050>;
1417defm DS_CMPST_F64           : DS_Real_gfx6_gfx7_gfx10<0x051>;
1418
1419defm DS_MIN_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x052>;
1420defm DS_MAX_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x053>;
1421defm DS_ADD_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x060>;
1422defm DS_SUB_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x061>;
1423defm DS_RSUB_RTN_U64        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x062>;
1424defm DS_INC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x063>;
1425defm DS_DEC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x064>;
1426defm DS_MIN_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x065>;
1427defm DS_MAX_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x066>;
1428defm DS_MIN_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x067>;
1429defm DS_MAX_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x068>;
1430defm DS_AND_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x069>;
1431defm DS_OR_RTN_B64          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06a>;
1432defm DS_XOR_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06b>;
1433defm DS_MSKOR_RTN_B64       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06c>;
1434
1435defm DS_WRXCHG_RTN_B64      : DS_Real_gfx6_gfx7_gfx10<0x06d>;
1436defm DS_WRXCHG2_RTN_B64     : DS_Real_gfx6_gfx7_gfx10<0x06e>;
1437defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06f>;
1438defm DS_CMPST_RTN_B64       : DS_Real_gfx6_gfx7_gfx10<0x070>;
1439defm DS_CMPST_RTN_F64       : DS_Real_gfx6_gfx7_gfx10<0x071>;
1440
1441defm DS_MIN_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x072>;
1442defm DS_MAX_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x073>;
1443
1444defm DS_READ_B64            : DS_Real_gfx6_gfx7_gfx10<0x076>;
1445defm DS_READ2_B64           : DS_Real_gfx6_gfx7_gfx10<0x077>;
1446defm DS_READ2ST64_B64       : DS_Real_gfx6_gfx7_gfx10<0x078>;
1447defm DS_ADD_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x080>;
1448defm DS_SUB_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x081>;
1449defm DS_RSUB_SRC2_U32       : DS_Real_gfx6_gfx7_gfx10<0x082>;
1450defm DS_INC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x083>;
1451defm DS_DEC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x084>;
1452defm DS_MIN_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x085>;
1453defm DS_MAX_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x086>;
1454defm DS_MIN_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x087>;
1455defm DS_MAX_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x088>;
1456defm DS_AND_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x089>;
1457defm DS_OR_SRC2_B32         : DS_Real_gfx6_gfx7_gfx10<0x08a>;
1458defm DS_XOR_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x08b>;
1459defm DS_WRITE_SRC2_B32      : DS_Real_gfx6_gfx7_gfx10<0x08d>;
1460defm DS_MIN_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x092>;
1461defm DS_MAX_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x093>;
1462defm DS_ADD_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c0>;
1463defm DS_SUB_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c1>;
1464defm DS_RSUB_SRC2_U64       : DS_Real_gfx6_gfx7_gfx10<0x0c2>;
1465defm DS_INC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c3>;
1466defm DS_DEC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c4>;
1467defm DS_MIN_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c5>;
1468defm DS_MAX_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c6>;
1469defm DS_MIN_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c7>;
1470defm DS_MAX_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c8>;
1471defm DS_AND_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0c9>;
1472defm DS_OR_SRC2_B64         : DS_Real_gfx6_gfx7_gfx10<0x0ca>;
1473defm DS_XOR_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0cb>;
1474defm DS_WRITE_SRC2_B64      : DS_Real_gfx6_gfx7_gfx10<0x0cd>;
1475defm DS_MIN_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d2>;
1476defm DS_MAX_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d3>;
1477
1478//===----------------------------------------------------------------------===//
1479// GFX8, GFX9 (VI).
1480//===----------------------------------------------------------------------===//
1481
1482class DS_Real_vi <bits<8> op, DS_Pseudo ps> :
1483  DS_Real <ps>,
1484  SIMCInstr <ps.Mnemonic, SIEncodingFamily.VI> {
1485  let AssemblerPredicate = isGFX8GFX9;
1486  let DecoderNamespace = "GFX8";
1487
1488  // encoding
1489  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1490  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1491  let Inst{16}    = !if(ps.has_gds, gds, ps.gdsValue);
1492  let Inst{24-17} = op;
1493  let Inst{25}    = acc;
1494  let Inst{31-26} = 0x36; // ds prefix
1495  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1496  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1497  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1498  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1499}
1500
1501def DS_ADD_U32_vi         : DS_Real_vi<0x0,  DS_ADD_U32>;
1502def DS_SUB_U32_vi         : DS_Real_vi<0x1,  DS_SUB_U32>;
1503def DS_RSUB_U32_vi        : DS_Real_vi<0x2,  DS_RSUB_U32>;
1504def DS_INC_U32_vi         : DS_Real_vi<0x3,  DS_INC_U32>;
1505def DS_DEC_U32_vi         : DS_Real_vi<0x4,  DS_DEC_U32>;
1506def DS_MIN_I32_vi         : DS_Real_vi<0x5,  DS_MIN_I32>;
1507def DS_MAX_I32_vi         : DS_Real_vi<0x6,  DS_MAX_I32>;
1508def DS_MIN_U32_vi         : DS_Real_vi<0x7,  DS_MIN_U32>;
1509def DS_MAX_U32_vi         : DS_Real_vi<0x8,  DS_MAX_U32>;
1510def DS_AND_B32_vi         : DS_Real_vi<0x9,  DS_AND_B32>;
1511def DS_OR_B32_vi          : DS_Real_vi<0xa,  DS_OR_B32>;
1512def DS_XOR_B32_vi         : DS_Real_vi<0xb,  DS_XOR_B32>;
1513def DS_MSKOR_B32_vi       : DS_Real_vi<0xc,  DS_MSKOR_B32>;
1514def DS_WRITE_B32_vi       : DS_Real_vi<0xd,  DS_WRITE_B32>;
1515def DS_WRITE2_B32_vi      : DS_Real_vi<0xe,  DS_WRITE2_B32>;
1516def DS_WRITE2ST64_B32_vi  : DS_Real_vi<0xf,  DS_WRITE2ST64_B32>;
1517def DS_CMPST_B32_vi       : DS_Real_vi<0x10, DS_CMPST_B32>;
1518def DS_CMPST_F32_vi       : DS_Real_vi<0x11, DS_CMPST_F32>;
1519def DS_MIN_F32_vi         : DS_Real_vi<0x12, DS_MIN_F32>;
1520def DS_MAX_F32_vi         : DS_Real_vi<0x13, DS_MAX_F32>;
1521def DS_NOP_vi             : DS_Real_vi<0x14, DS_NOP>;
1522def DS_ADD_F32_vi         : DS_Real_vi<0x15, DS_ADD_F32>;
1523def DS_GWS_INIT_vi        : DS_Real_vi<0x99, DS_GWS_INIT>;
1524def DS_GWS_SEMA_V_vi      : DS_Real_vi<0x9a, DS_GWS_SEMA_V>;
1525def DS_GWS_SEMA_BR_vi     : DS_Real_vi<0x9b, DS_GWS_SEMA_BR>;
1526def DS_GWS_SEMA_P_vi      : DS_Real_vi<0x9c, DS_GWS_SEMA_P>;
1527def DS_GWS_BARRIER_vi     : DS_Real_vi<0x9d, DS_GWS_BARRIER>;
1528def DS_WRITE_ADDTID_B32_vi : DS_Real_vi<0x1d, DS_WRITE_ADDTID_B32>;
1529def DS_WRITE_B8_vi        : DS_Real_vi<0x1e, DS_WRITE_B8>;
1530def DS_WRITE_B16_vi       : DS_Real_vi<0x1f, DS_WRITE_B16>;
1531def DS_ADD_RTN_U32_vi     : DS_Real_vi<0x20, DS_ADD_RTN_U32>;
1532def DS_SUB_RTN_U32_vi     : DS_Real_vi<0x21, DS_SUB_RTN_U32>;
1533def DS_RSUB_RTN_U32_vi    : DS_Real_vi<0x22, DS_RSUB_RTN_U32>;
1534def DS_INC_RTN_U32_vi     : DS_Real_vi<0x23, DS_INC_RTN_U32>;
1535def DS_DEC_RTN_U32_vi     : DS_Real_vi<0x24, DS_DEC_RTN_U32>;
1536def DS_MIN_RTN_I32_vi     : DS_Real_vi<0x25, DS_MIN_RTN_I32>;
1537def DS_MAX_RTN_I32_vi     : DS_Real_vi<0x26, DS_MAX_RTN_I32>;
1538def DS_MIN_RTN_U32_vi     : DS_Real_vi<0x27, DS_MIN_RTN_U32>;
1539def DS_MAX_RTN_U32_vi     : DS_Real_vi<0x28, DS_MAX_RTN_U32>;
1540def DS_AND_RTN_B32_vi     : DS_Real_vi<0x29, DS_AND_RTN_B32>;
1541def DS_OR_RTN_B32_vi      : DS_Real_vi<0x2a, DS_OR_RTN_B32>;
1542def DS_XOR_RTN_B32_vi     : DS_Real_vi<0x2b, DS_XOR_RTN_B32>;
1543def DS_MSKOR_RTN_B32_vi   : DS_Real_vi<0x2c, DS_MSKOR_RTN_B32>;
1544def DS_WRXCHG_RTN_B32_vi  : DS_Real_vi<0x2d, DS_WRXCHG_RTN_B32>;
1545def DS_WRXCHG2_RTN_B32_vi : DS_Real_vi<0x2e, DS_WRXCHG2_RTN_B32>;
1546def DS_WRXCHG2ST64_RTN_B32_vi : DS_Real_vi<0x2f, DS_WRXCHG2ST64_RTN_B32>;
1547def DS_CMPST_RTN_B32_vi   : DS_Real_vi<0x30, DS_CMPST_RTN_B32>;
1548def DS_CMPST_RTN_F32_vi   : DS_Real_vi<0x31, DS_CMPST_RTN_F32>;
1549def DS_MIN_RTN_F32_vi     : DS_Real_vi<0x32, DS_MIN_RTN_F32>;
1550def DS_MAX_RTN_F32_vi     : DS_Real_vi<0x33, DS_MAX_RTN_F32>;
1551def DS_WRAP_RTN_B32_vi    : DS_Real_vi<0x34, DS_WRAP_RTN_B32>;
1552def DS_ADD_RTN_F32_vi     : DS_Real_vi<0x35, DS_ADD_RTN_F32>;
1553def DS_READ_B32_vi        : DS_Real_vi<0x36, DS_READ_B32>;
1554def DS_READ2_B32_vi       : DS_Real_vi<0x37, DS_READ2_B32>;
1555def DS_READ2ST64_B32_vi   : DS_Real_vi<0x38, DS_READ2ST64_B32>;
1556def DS_READ_I8_vi         : DS_Real_vi<0x39, DS_READ_I8>;
1557def DS_READ_U8_vi         : DS_Real_vi<0x3a, DS_READ_U8>;
1558def DS_READ_I16_vi        : DS_Real_vi<0x3b, DS_READ_I16>;
1559def DS_READ_U16_vi        : DS_Real_vi<0x3c, DS_READ_U16>;
1560def DS_READ_ADDTID_B32_vi : DS_Real_vi<0xb6, DS_READ_ADDTID_B32>;
1561def DS_CONSUME_vi         : DS_Real_vi<0xbd, DS_CONSUME>;
1562def DS_APPEND_vi          : DS_Real_vi<0xbe, DS_APPEND>;
1563def DS_ORDERED_COUNT_vi   : DS_Real_vi<0xbf, DS_ORDERED_COUNT>;
1564def DS_SWIZZLE_B32_vi     : DS_Real_vi<0x3d, DS_SWIZZLE_B32>;
1565def DS_PERMUTE_B32_vi     : DS_Real_vi<0x3e, DS_PERMUTE_B32>;
1566def DS_BPERMUTE_B32_vi    : DS_Real_vi<0x3f, DS_BPERMUTE_B32>;
1567
1568def DS_ADD_U64_vi         : DS_Real_vi<0x40, DS_ADD_U64>;
1569def DS_SUB_U64_vi         : DS_Real_vi<0x41, DS_SUB_U64>;
1570def DS_RSUB_U64_vi        : DS_Real_vi<0x42, DS_RSUB_U64>;
1571def DS_INC_U64_vi         : DS_Real_vi<0x43, DS_INC_U64>;
1572def DS_DEC_U64_vi         : DS_Real_vi<0x44, DS_DEC_U64>;
1573def DS_MIN_I64_vi         : DS_Real_vi<0x45, DS_MIN_I64>;
1574def DS_MAX_I64_vi         : DS_Real_vi<0x46, DS_MAX_I64>;
1575def DS_MIN_U64_vi         : DS_Real_vi<0x47, DS_MIN_U64>;
1576def DS_MAX_U64_vi         : DS_Real_vi<0x48, DS_MAX_U64>;
1577def DS_AND_B64_vi         : DS_Real_vi<0x49, DS_AND_B64>;
1578def DS_OR_B64_vi          : DS_Real_vi<0x4a, DS_OR_B64>;
1579def DS_XOR_B64_vi         : DS_Real_vi<0x4b, DS_XOR_B64>;
1580def DS_MSKOR_B64_vi       : DS_Real_vi<0x4c, DS_MSKOR_B64>;
1581def DS_WRITE_B64_vi       : DS_Real_vi<0x4d, DS_WRITE_B64>;
1582def DS_WRITE2_B64_vi      : DS_Real_vi<0x4E, DS_WRITE2_B64>;
1583def DS_WRITE2ST64_B64_vi  : DS_Real_vi<0x4f, DS_WRITE2ST64_B64>;
1584def DS_CMPST_B64_vi       : DS_Real_vi<0x50, DS_CMPST_B64>;
1585def DS_CMPST_F64_vi       : DS_Real_vi<0x51, DS_CMPST_F64>;
1586def DS_MIN_F64_vi         : DS_Real_vi<0x52, DS_MIN_F64>;
1587def DS_MAX_F64_vi         : DS_Real_vi<0x53, DS_MAX_F64>;
1588
1589def DS_WRITE_B8_D16_HI_vi  : DS_Real_vi<0x54, DS_WRITE_B8_D16_HI>;
1590def DS_WRITE_B16_D16_HI_vi : DS_Real_vi<0x55, DS_WRITE_B16_D16_HI>;
1591
1592def DS_READ_U8_D16_vi     : DS_Real_vi<0x56, DS_READ_U8_D16>;
1593def DS_READ_U8_D16_HI_vi  : DS_Real_vi<0x57, DS_READ_U8_D16_HI>;
1594def DS_READ_I8_D16_vi     : DS_Real_vi<0x58, DS_READ_I8_D16>;
1595def DS_READ_I8_D16_HI_vi  : DS_Real_vi<0x59, DS_READ_I8_D16_HI>;
1596def DS_READ_U16_D16_vi    : DS_Real_vi<0x5a, DS_READ_U16_D16>;
1597def DS_READ_U16_D16_HI_vi : DS_Real_vi<0x5b, DS_READ_U16_D16_HI>;
1598
1599def DS_ADD_RTN_U64_vi     : DS_Real_vi<0x60, DS_ADD_RTN_U64>;
1600def DS_SUB_RTN_U64_vi     : DS_Real_vi<0x61, DS_SUB_RTN_U64>;
1601def DS_RSUB_RTN_U64_vi    : DS_Real_vi<0x62, DS_RSUB_RTN_U64>;
1602def DS_INC_RTN_U64_vi     : DS_Real_vi<0x63, DS_INC_RTN_U64>;
1603def DS_DEC_RTN_U64_vi     : DS_Real_vi<0x64, DS_DEC_RTN_U64>;
1604def DS_MIN_RTN_I64_vi     : DS_Real_vi<0x65, DS_MIN_RTN_I64>;
1605def DS_MAX_RTN_I64_vi     : DS_Real_vi<0x66, DS_MAX_RTN_I64>;
1606def DS_MIN_RTN_U64_vi     : DS_Real_vi<0x67, DS_MIN_RTN_U64>;
1607def DS_MAX_RTN_U64_vi     : DS_Real_vi<0x68, DS_MAX_RTN_U64>;
1608def DS_AND_RTN_B64_vi     : DS_Real_vi<0x69, DS_AND_RTN_B64>;
1609def DS_OR_RTN_B64_vi      : DS_Real_vi<0x6a, DS_OR_RTN_B64>;
1610def DS_XOR_RTN_B64_vi     : DS_Real_vi<0x6b, DS_XOR_RTN_B64>;
1611def DS_MSKOR_RTN_B64_vi   : DS_Real_vi<0x6c, DS_MSKOR_RTN_B64>;
1612def DS_WRXCHG_RTN_B64_vi  : DS_Real_vi<0x6d, DS_WRXCHG_RTN_B64>;
1613def DS_WRXCHG2_RTN_B64_vi : DS_Real_vi<0x6e, DS_WRXCHG2_RTN_B64>;
1614def DS_WRXCHG2ST64_RTN_B64_vi : DS_Real_vi<0x6f, DS_WRXCHG2ST64_RTN_B64>;
1615def DS_CONDXCHG32_RTN_B64_vi   : DS_Real_vi<0x7e, DS_CONDXCHG32_RTN_B64>;
1616def DS_GWS_SEMA_RELEASE_ALL_vi : DS_Real_vi<0x98, DS_GWS_SEMA_RELEASE_ALL>;
1617def DS_CMPST_RTN_B64_vi   : DS_Real_vi<0x70, DS_CMPST_RTN_B64>;
1618def DS_CMPST_RTN_F64_vi   : DS_Real_vi<0x71, DS_CMPST_RTN_F64>;
1619def DS_MIN_RTN_F64_vi     : DS_Real_vi<0x72, DS_MIN_RTN_F64>;
1620def DS_MAX_RTN_F64_vi     : DS_Real_vi<0x73, DS_MAX_RTN_F64>;
1621
1622def DS_READ_B64_vi        : DS_Real_vi<0x76, DS_READ_B64>;
1623def DS_READ2_B64_vi       : DS_Real_vi<0x77, DS_READ2_B64>;
1624def DS_READ2ST64_B64_vi   : DS_Real_vi<0x78, DS_READ2ST64_B64>;
1625
1626def DS_ADD_SRC2_U32_vi    : DS_Real_vi<0x80, DS_ADD_SRC2_U32>;
1627def DS_SUB_SRC2_U32_vi    : DS_Real_vi<0x81, DS_SUB_SRC2_U32>;
1628def DS_RSUB_SRC2_U32_vi   : DS_Real_vi<0x82, DS_RSUB_SRC2_U32>;
1629def DS_INC_SRC2_U32_vi    : DS_Real_vi<0x83, DS_INC_SRC2_U32>;
1630def DS_DEC_SRC2_U32_vi    : DS_Real_vi<0x84, DS_DEC_SRC2_U32>;
1631def DS_MIN_SRC2_I32_vi    : DS_Real_vi<0x85, DS_MIN_SRC2_I32>;
1632def DS_MAX_SRC2_I32_vi    : DS_Real_vi<0x86, DS_MAX_SRC2_I32>;
1633def DS_MIN_SRC2_U32_vi    : DS_Real_vi<0x87, DS_MIN_SRC2_U32>;
1634def DS_MAX_SRC2_U32_vi    : DS_Real_vi<0x88, DS_MAX_SRC2_U32>;
1635def DS_AND_SRC2_B32_vi    : DS_Real_vi<0x89, DS_AND_SRC2_B32>;
1636def DS_OR_SRC2_B32_vi     : DS_Real_vi<0x8a, DS_OR_SRC2_B32>;
1637def DS_XOR_SRC2_B32_vi    : DS_Real_vi<0x8b, DS_XOR_SRC2_B32>;
1638def DS_WRITE_SRC2_B32_vi  : DS_Real_vi<0x8d, DS_WRITE_SRC2_B32>;
1639def DS_MIN_SRC2_F32_vi    : DS_Real_vi<0x92, DS_MIN_SRC2_F32>;
1640def DS_MAX_SRC2_F32_vi    : DS_Real_vi<0x93, DS_MAX_SRC2_F32>;
1641def DS_ADD_SRC2_F32_vi    : DS_Real_vi<0x95, DS_ADD_SRC2_F32>;
1642def DS_ADD_SRC2_U64_vi    : DS_Real_vi<0xc0, DS_ADD_SRC2_U64>;
1643def DS_SUB_SRC2_U64_vi    : DS_Real_vi<0xc1, DS_SUB_SRC2_U64>;
1644def DS_RSUB_SRC2_U64_vi   : DS_Real_vi<0xc2, DS_RSUB_SRC2_U64>;
1645def DS_INC_SRC2_U64_vi    : DS_Real_vi<0xc3, DS_INC_SRC2_U64>;
1646def DS_DEC_SRC2_U64_vi    : DS_Real_vi<0xc4, DS_DEC_SRC2_U64>;
1647def DS_MIN_SRC2_I64_vi    : DS_Real_vi<0xc5, DS_MIN_SRC2_I64>;
1648def DS_MAX_SRC2_I64_vi    : DS_Real_vi<0xc6, DS_MAX_SRC2_I64>;
1649def DS_MIN_SRC2_U64_vi    : DS_Real_vi<0xc7, DS_MIN_SRC2_U64>;
1650def DS_MAX_SRC2_U64_vi    : DS_Real_vi<0xc8, DS_MAX_SRC2_U64>;
1651def DS_AND_SRC2_B64_vi    : DS_Real_vi<0xc9, DS_AND_SRC2_B64>;
1652def DS_OR_SRC2_B64_vi     : DS_Real_vi<0xca, DS_OR_SRC2_B64>;
1653def DS_XOR_SRC2_B64_vi    : DS_Real_vi<0xcb, DS_XOR_SRC2_B64>;
1654def DS_WRITE_SRC2_B64_vi  : DS_Real_vi<0xcd, DS_WRITE_SRC2_B64>;
1655def DS_MIN_SRC2_F64_vi    : DS_Real_vi<0xd2, DS_MIN_SRC2_F64>;
1656def DS_MAX_SRC2_F64_vi    : DS_Real_vi<0xd3, DS_MAX_SRC2_F64>;
1657def DS_WRITE_B96_vi       : DS_Real_vi<0xde, DS_WRITE_B96>;
1658def DS_WRITE_B128_vi      : DS_Real_vi<0xdf, DS_WRITE_B128>;
1659def DS_READ_B96_vi        : DS_Real_vi<0xfe, DS_READ_B96>;
1660def DS_READ_B128_vi       : DS_Real_vi<0xff, DS_READ_B128>;
1661
1662let SubtargetPredicate = isGFX90APlus in {
1663  def DS_ADD_F64_vi     : DS_Real_vi<0x5c, DS_ADD_F64>;
1664  def DS_ADD_RTN_F64_vi : DS_Real_vi<0x7c, DS_ADD_RTN_F64>;
1665} // End SubtargetPredicate = isGFX90APlus
1666
1667let SubtargetPredicate = isGFX940Plus in {
1668  def DS_PK_ADD_F16_vi     : DS_Real_vi<0x17, DS_PK_ADD_F16>;
1669  def DS_PK_ADD_RTN_F16_vi : DS_Real_vi<0xb7, DS_PK_ADD_RTN_F16>;
1670  def DS_PK_ADD_BF16_vi     : DS_Real_vi<0x18, DS_PK_ADD_BF16>;
1671  def DS_PK_ADD_RTN_BF16_vi : DS_Real_vi<0xb8, DS_PK_ADD_RTN_BF16>;
1672} // End SubtargetPredicate = isGFX940Plus
1673