xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/DSInstructions.td (revision 9f23cbd6cae82fd77edfad7173432fa8dccd0a95) 
1//===-- DSInstructions.td - DS Instruction Definitions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
10  InstSI <outs, ins, "", pattern>,
11  SIMCInstr <opName, SIEncodingFamily.NONE> {
12
13  let LGKM_CNT = 1;
14  let DS = 1;
15  let Size = 8;
16  let UseNamedOperandTable = 1;
17
18  // Most instruction load and store data, so set this as the default.
19  let mayLoad = 1;
20  let mayStore = 1;
21  let maybeAtomic = 1;
22
23  let hasSideEffects = 0;
24  let SchedRW = [WriteLDS];
25
26  let isPseudo = 1;
27  let isCodeGenOnly = 1;
28
29  let AsmMatchConverter = "cvtDS";
30
31  string Mnemonic = opName;
32  string AsmOperands = asmOps;
33
34  // Well these bits a kind of hack because it would be more natural
35  // to test "outs" and "ins" dags for the presence of particular operands
36  bits<1> has_vdst = 1;
37  bits<1> has_addr = 1;
38  bits<1> has_data0 = 1;
39  bits<1> has_data1 = 1;
40
41  bits<1> has_gws_data0 = 0; // data0 is encoded as addr
42
43  bits<1> has_offset  = 1; // has "offset" that should be split to offset0,1
44  bits<1> has_offset0 = 1;
45  bits<1> has_offset1 = 1;
46
47  bits<1> has_gds = 1;
48  bits<1> gdsValue = 0; // if has_gds == 0 set gds to this value
49
50  bits<1> has_m0_read = 1;
51
52  let Uses = !if(has_m0_read, [M0, EXEC], [EXEC]);
53}
54
55class DS_Real <DS_Pseudo ps, string opName = ps.Mnemonic> :
56  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands>,
57  Enc64 {
58
59  let isPseudo = 0;
60  let isCodeGenOnly = 0;
61  let LGKM_CNT = 1;
62  let DS = 1;
63  let UseNamedOperandTable = 1;
64
65  // copy relevant pseudo op flags
66  let SubtargetPredicate = ps.SubtargetPredicate;
67  let OtherPredicates    = ps.OtherPredicates;
68  let AsmMatchConverter  = ps.AsmMatchConverter;
69  let SchedRW            = ps.SchedRW;
70  let mayLoad            = ps.mayLoad;
71  let mayStore           = ps.mayStore;
72  let IsAtomicRet        = ps.IsAtomicRet;
73  let IsAtomicNoRet      = ps.IsAtomicNoRet;
74
75  let Constraints = ps.Constraints;
76  let DisableEncoding = ps.DisableEncoding;
77
78  // encoding fields
79  bits<10> vdst;
80  bits<1> gds;
81  bits<8> addr;
82  bits<10> data0;
83  bits<10> data1;
84  bits<8> offset0;
85  bits<8> offset1;
86
87  bits<16> offset;
88  let offset0 = !if(ps.has_offset, offset{7-0}, ?);
89  let offset1 = !if(ps.has_offset, offset{15-8}, ?);
90
91  bits<1> acc = !if(ps.has_vdst, vdst{9},
92                    !if(!or(ps.has_data0, ps.has_gws_data0), data0{9}, 0));
93}
94
95// DS Pseudo instructions
96
97class DS_0A1D_NORET<string opName, RegisterClass rc = VGPR_32>
98: DS_Pseudo<opName,
99  (outs),
100  (ins getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
101  " $data0$offset$gds"> {
102
103  let has_addr = 0;
104  let has_data1 = 0;
105  let has_vdst = 0;
106}
107
108class DS_1A1D_NORET<string opName, RegisterClass rc = VGPR_32>
109: DS_Pseudo<opName,
110  (outs),
111  (ins VGPR_32:$addr, getLdStRegisterOperand<rc>.ret:$data0, offset:$offset, gds:$gds),
112  " $addr, $data0$offset$gds"> {
113
114  let has_data1 = 0;
115  let has_vdst = 0;
116  let IsAtomicNoRet = 1;
117}
118
119multiclass DS_1A1D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
120  def "" : DS_1A1D_NORET<opName, rc>,
121           AtomicNoRet<opName, 0>;
122
123  let has_m0_read = 0 in {
124    def _gfx9 : DS_1A1D_NORET<opName, rc>,
125                AtomicNoRet<opName#"_gfx9", 0>;
126  }
127}
128
129multiclass DS_1A1D_NORET_mc_gfx9<string opName, RegisterClass rc = VGPR_32> {
130  let has_m0_read = 0 in {
131    def "" : DS_1A1D_NORET<opName, rc>,
132                AtomicNoRet<opName, 0>;
133  }
134}
135
136class DS_1A2D_NORET<string opName, RegisterClass rc = VGPR_32,
137                    RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
138: DS_Pseudo<opName,
139  (outs),
140  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1, offset:$offset, gds:$gds),
141  " $addr, $data0, $data1$offset$gds"> {
142
143  let has_vdst = 0;
144  let IsAtomicNoRet = 1;
145}
146
147multiclass DS_1A2D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
148  def "" : DS_1A2D_NORET<opName, rc>,
149           AtomicNoRet<opName, 0>;
150
151  let has_m0_read = 0 in {
152    def _gfx9 : DS_1A2D_NORET<opName, rc>,
153                AtomicNoRet<opName#"_gfx9", 0>;
154  }
155}
156
157class DS_1A2D_Off8_NORET <string opName, RegisterClass rc = VGPR_32,
158                          RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
159: DS_Pseudo<opName,
160  (outs),
161  (ins VGPR_32:$addr, data_op:$data0, data_op:$data1,
162       offset0:$offset0, offset1:$offset1, gds:$gds),
163  " $addr, $data0, $data1$offset0$offset1$gds"> {
164
165  let has_vdst = 0;
166  let has_offset = 0;
167  let AsmMatchConverter = "cvtDSOffset01";
168}
169
170multiclass DS_1A2D_Off8_NORET_mc <string opName, RegisterClass rc = VGPR_32> {
171  def "" : DS_1A2D_Off8_NORET<opName, rc>;
172
173  let has_m0_read = 0 in {
174    def _gfx9 : DS_1A2D_Off8_NORET<opName, rc>;
175  }
176}
177
178class DS_0A1D_RET_GDS<string opName, RegisterClass rc = VGPR_32, RegisterClass src = rc,
179                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
180                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
181: DS_Pseudo<opName,
182  (outs dst_op:$vdst),
183  (ins src_op:$data0, offset:$offset),
184  " $vdst, $data0$offset gds"> {
185
186  let has_addr = 0;
187  let has_data1 = 0;
188  let has_gds = 0;
189  let gdsValue = 1;
190  let AsmMatchConverter = "cvtDSGds";
191  let hasSideEffects = 1;
192}
193
194class DS_1A1D_RET <string opName, RegisterClass rc = VGPR_32,
195                  RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
196: DS_Pseudo<opName,
197  (outs data_op:$vdst),
198  (ins VGPR_32:$addr, data_op:$data0, offset:$offset, gds:$gds),
199  " $vdst, $addr, $data0$offset$gds"> {
200
201  let hasPostISelHook = 1;
202  let has_data1 = 0;
203  let IsAtomicRet = 1;
204}
205
206multiclass DS_1A1D_RET_mc <string opName, RegisterClass rc = VGPR_32,
207                           string NoRetOp = ""> {
208  def "" : DS_1A1D_RET<opName, rc>,
209    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
210
211  let has_m0_read = 0 in {
212    def _gfx9 : DS_1A1D_RET<opName, rc>,
213      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp#"_gfx9"),
214                  !ne(NoRetOp, "")>;
215  }
216}
217
218multiclass DS_1A1D_RET_mc_gfx9 <string opName, RegisterClass rc = VGPR_32,
219                                string NoRetOp = ""> {
220  let has_m0_read = 0 in {
221    def "" : DS_1A1D_RET<opName, rc>,
222      AtomicNoRet<!if(!eq(NoRetOp, ""), "", NoRetOp),
223                  !if(!eq(NoRetOp, ""), 0, 1)>;
224  }
225}
226
227class DS_1A2D_RET<string opName,
228                  RegisterClass rc = VGPR_32,
229                  RegisterClass src = rc,
230                  RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
231                  RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
232: DS_Pseudo<opName,
233  (outs dst_op:$vdst),
234  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset:$offset, gds:$gds),
235  " $vdst, $addr, $data0, $data1$offset$gds"> {
236
237  let hasPostISelHook = 1;
238  let IsAtomicRet = 1;
239}
240
241multiclass DS_1A2D_RET_mc<string opName,
242                          RegisterClass rc = VGPR_32,
243                          string NoRetOp = "",
244                          RegisterClass src = rc> {
245  def "" : DS_1A2D_RET<opName, rc, src>,
246    AtomicNoRet<NoRetOp, !ne(NoRetOp, "")>;
247
248  let has_m0_read = 0 in {
249    def _gfx9 : DS_1A2D_RET<opName, rc, src>,
250      AtomicNoRet<NoRetOp#"_gfx9", !ne(NoRetOp, "")>;
251  }
252}
253
254class DS_1A2D_Off8_RET<string opName,
255                       RegisterClass rc = VGPR_32,
256                       RegisterClass src = rc,
257                       RegisterOperand dst_op = getLdStRegisterOperand<rc>.ret,
258                       RegisterOperand src_op = getLdStRegisterOperand<src>.ret>
259: DS_Pseudo<opName,
260  (outs dst_op:$vdst),
261  (ins VGPR_32:$addr, src_op:$data0, src_op:$data1, offset0:$offset0, offset1:$offset1, gds:$gds),
262  " $vdst, $addr, $data0, $data1$offset0$offset1$gds"> {
263
264  let has_offset = 0;
265  let AsmMatchConverter = "cvtDSOffset01";
266
267  let hasPostISelHook = 1;
268}
269
270multiclass DS_1A2D_Off8_RET_mc<string opName,
271                               RegisterClass rc = VGPR_32,
272                               RegisterClass src = rc> {
273  def "" : DS_1A2D_Off8_RET<opName, rc, src>;
274
275  let has_m0_read = 0 in {
276    def _gfx9 : DS_1A2D_Off8_RET<opName, rc, src>;
277  }
278}
279
280class DS_BVH_STACK<string opName>
281: DS_Pseudo<opName,
282  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst, VGPR_32:$addr),
283  (ins VGPR_32:$addr_in, getLdStRegisterOperand<VGPR_32>.ret:$data0, VReg_128:$data1, offset:$offset),
284  " $vdst, $addr, $data0, $data1$offset"> {
285  let Constraints = "$addr = $addr_in";
286  let DisableEncoding = "$addr_in";
287  let has_gds = 0;
288  let gdsValue = 0;
289  // TODO: Use MMOs in the LDS address space instead of hasSideEffects = 1.
290  let hasSideEffects = 1;
291  let SchedRW = [WriteLDS, WriteLDS];
292}
293
294class DS_1A_RET<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset,
295                RegisterOperand data_op = getLdStRegisterOperand<rc>.ret>
296: DS_Pseudo<opName,
297  (outs data_op:$vdst),
298  !if(HasTiedOutput,
299    (ins VGPR_32:$addr, ofs:$offset, gds:$gds, data_op:$vdst_in),
300    (ins VGPR_32:$addr, ofs:$offset, gds:$gds)),
301  " $vdst, $addr$offset$gds"> {
302  let Constraints = !if(HasTiedOutput, "$vdst = $vdst_in", "");
303  let DisableEncoding = !if(HasTiedOutput, "$vdst_in", "");
304  let has_data0 = 0;
305  let has_data1 = 0;
306}
307
308multiclass DS_1A_RET_mc<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = offset> {
309  def "" : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
310
311  let has_m0_read = 0 in {
312    def _gfx9 : DS_1A_RET<opName, rc, HasTiedOutput, ofs>;
313  }
314}
315
316class DS_1A_RET_Tied<string opName, RegisterClass rc = VGPR_32> :
317  DS_1A_RET<opName, rc, 1>;
318
319class DS_1A_Off8_RET <string opName, RegisterClass rc = VGPR_32>
320: DS_Pseudo<opName,
321  (outs getLdStRegisterOperand<rc>.ret:$vdst),
322  (ins VGPR_32:$addr, offset0:$offset0, offset1:$offset1, gds:$gds),
323  " $vdst, $addr$offset0$offset1$gds"> {
324
325  let has_offset = 0;
326  let has_data0 = 0;
327  let has_data1 = 0;
328  let AsmMatchConverter = "cvtDSOffset01";
329}
330
331multiclass DS_1A_Off8_RET_mc <string opName, RegisterClass rc = VGPR_32> {
332  def "" : DS_1A_Off8_RET<opName, rc>;
333
334  let has_m0_read = 0 in {
335    def _gfx9 : DS_1A_Off8_RET<opName, rc>;
336  }
337}
338
339class DS_1A_RET_GDS <string opName> : DS_Pseudo<opName,
340  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
341  (ins VGPR_32:$addr, offset:$offset),
342  " $vdst, $addr$offset gds"> {
343
344  let has_data0 = 0;
345  let has_data1 = 0;
346  let has_gds = 0;
347  let gdsValue = 1;
348  let AsmMatchConverter = "cvtDSGds";
349}
350
351class DS_0A_RET <string opName> : DS_Pseudo<opName,
352  (outs getLdStRegisterOperand<VGPR_32>.ret:$vdst),
353  (ins offset:$offset, gds:$gds),
354  " $vdst$offset$gds"> {
355
356  let mayLoad = 1;
357  let mayStore = 1;
358
359  let has_addr = 0;
360  let has_data0 = 0;
361  let has_data1 = 0;
362}
363
364class DS_1A <string opName> : DS_Pseudo<opName,
365  (outs),
366  (ins VGPR_32:$addr, offset:$offset, gds:$gds),
367  " $addr$offset$gds"> {
368
369  let mayLoad = 1;
370  let mayStore = 1;
371
372  let has_vdst = 0;
373  let has_data0 = 0;
374  let has_data1 = 0;
375}
376
377multiclass DS_1A_mc <string opName> {
378  def "" : DS_1A<opName>;
379
380  let has_m0_read = 0 in {
381    def _gfx9 : DS_1A<opName>;
382  }
383}
384
385
386class DS_GWS <string opName, dag ins, string asmOps>
387: DS_Pseudo<opName, (outs), ins, asmOps> {
388
389  let has_vdst  = 0;
390  let has_addr  = 0;
391  let has_data0 = 0;
392  let has_data1 = 0;
393
394  let has_gds   = 0;
395  let gdsValue  = 1;
396  let AsmMatchConverter = "cvtDSGds";
397}
398
399class DS_GWS_0D <string opName>
400: DS_GWS<opName,
401  (ins offset:$offset), "$offset gds"> {
402  let hasSideEffects = 1;
403}
404
405class DS_GWS_1D <string opName>
406: DS_GWS<opName,
407  (ins getLdStRegisterOperand<VGPR_32>.ret:$data0, offset:$offset),
408  " $data0$offset gds"> {
409
410  let has_gws_data0 = 1;
411  let hasSideEffects = 1;
412}
413
414class DS_VOID <string opName> : DS_Pseudo<opName,
415  (outs), (ins), ""> {
416  let mayLoad = 0;
417  let mayStore = 0;
418  let hasSideEffects = 1;
419  let UseNamedOperandTable = 0;
420  let AsmMatchConverter = "";
421
422  let has_vdst = 0;
423  let has_addr = 0;
424  let has_data0 = 0;
425  let has_data1 = 0;
426  let has_offset = 0;
427  let has_offset0 = 0;
428  let has_offset1 = 0;
429  let has_gds = 0;
430}
431
432class DS_1A1D_PERMUTE <string opName, SDPatternOperator node = null_frag,
433                       RegisterOperand data_op = getLdStRegisterOperand<VGPR_32>.ret>
434: DS_Pseudo<opName,
435  (outs data_op:$vdst),
436  (ins VGPR_32:$addr, data_op:$data0, offset:$offset),
437  " $vdst, $addr, $data0$offset",
438  [(set i32:$vdst,
439   (node (DS1Addr1Offset i32:$addr, i16:$offset), i32:$data0))] > {
440
441  let mayLoad = 0;
442  let mayStore = 0;
443  let isConvergent = 1;
444
445  let has_data1 = 0;
446  let has_gds = 0;
447}
448
449defm DS_ADD_U32       : DS_1A1D_NORET_mc<"ds_add_u32">;
450defm DS_SUB_U32       : DS_1A1D_NORET_mc<"ds_sub_u32">;
451defm DS_RSUB_U32      : DS_1A1D_NORET_mc<"ds_rsub_u32">;
452defm DS_INC_U32       : DS_1A1D_NORET_mc<"ds_inc_u32">;
453defm DS_DEC_U32       : DS_1A1D_NORET_mc<"ds_dec_u32">;
454defm DS_MIN_I32       : DS_1A1D_NORET_mc<"ds_min_i32">;
455defm DS_MAX_I32       : DS_1A1D_NORET_mc<"ds_max_i32">;
456defm DS_MIN_U32       : DS_1A1D_NORET_mc<"ds_min_u32">;
457defm DS_MAX_U32       : DS_1A1D_NORET_mc<"ds_max_u32">;
458defm DS_AND_B32       : DS_1A1D_NORET_mc<"ds_and_b32">;
459defm DS_OR_B32        : DS_1A1D_NORET_mc<"ds_or_b32">;
460defm DS_XOR_B32       : DS_1A1D_NORET_mc<"ds_xor_b32">;
461
462let SubtargetPredicate = HasLDSFPAtomicAdd in {
463defm DS_ADD_F32       : DS_1A1D_NORET_mc<"ds_add_f32">;
464}
465
466defm DS_MIN_F32       : DS_1A1D_NORET_mc<"ds_min_f32">;
467defm DS_MAX_F32       : DS_1A1D_NORET_mc<"ds_max_f32">;
468
469let mayLoad = 0 in {
470defm DS_WRITE_B8      : DS_1A1D_NORET_mc<"ds_write_b8">;
471defm DS_WRITE_B16     : DS_1A1D_NORET_mc<"ds_write_b16">;
472defm DS_WRITE_B32     : DS_1A1D_NORET_mc<"ds_write_b32">;
473defm DS_WRITE2_B32    : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">;
474defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">;
475
476
477let has_m0_read = 0 in {
478
479let SubtargetPredicate = HasD16LoadStore in {
480def DS_WRITE_B8_D16_HI  : DS_1A1D_NORET<"ds_write_b8_d16_hi">;
481def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;
482}
483
484} // End has_m0_read = 0
485
486let SubtargetPredicate = HasDSAddTid in {
487def DS_WRITE_ADDTID_B32 : DS_0A1D_NORET<"ds_write_addtid_b32">;
488}
489
490} // End mayLoad = 0
491
492let SubtargetPredicate = isGFX90APlus in {
493  defm DS_ADD_F64     : DS_1A1D_NORET_mc_gfx9<"ds_add_f64", VReg_64>;
494  defm DS_ADD_RTN_F64 : DS_1A1D_RET_mc_gfx9<"ds_add_rtn_f64", VReg_64, "ds_add_f64">;
495} // End SubtargetPredicate = isGFX90APlus
496
497let SubtargetPredicate = isGFX940Plus in {
498  defm DS_PK_ADD_F16      : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_f16">;
499  defm DS_PK_ADD_RTN_F16  : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_f16", VGPR_32, "ds_pk_add_f16">;
500  defm DS_PK_ADD_BF16     : DS_1A1D_NORET_mc_gfx9<"ds_pk_add_bf16">;
501  defm DS_PK_ADD_RTN_BF16 : DS_1A1D_RET_mc_gfx9<"ds_pk_add_rtn_bf16", VGPR_32, "ds_pk_add_bf16">;
502} // End SubtargetPredicate = isGFX940Plus
503
504defm DS_CMPSTORE_B32     : DS_1A2D_NORET_mc<"ds_cmpstore_b32">;
505defm DS_CMPSTORE_F32     : DS_1A2D_NORET_mc<"ds_cmpstore_f32">;
506defm DS_CMPSTORE_B64     : DS_1A2D_NORET_mc<"ds_cmpstore_b64", VReg_64>;
507defm DS_CMPSTORE_F64     : DS_1A2D_NORET_mc<"ds_cmpstore_f64", VReg_64>;
508defm DS_CMPSTORE_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b32", VGPR_32, "ds_cmpstore_b32">;
509defm DS_CMPSTORE_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f32", VGPR_32, "ds_cmpstore_f32">;
510defm DS_CMPSTORE_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_b64", VReg_64, "ds_cmpstore_b64">;
511defm DS_CMPSTORE_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpstore_rtn_f64", VReg_64, "ds_cmpstore_f64">;
512
513defm DS_MSKOR_B32     : DS_1A2D_NORET_mc<"ds_mskor_b32">;
514defm DS_CMPST_B32     : DS_1A2D_NORET_mc<"ds_cmpst_b32">;
515defm DS_CMPST_F32     : DS_1A2D_NORET_mc<"ds_cmpst_f32">;
516
517defm DS_ADD_U64       : DS_1A1D_NORET_mc<"ds_add_u64", VReg_64>;
518defm DS_SUB_U64       : DS_1A1D_NORET_mc<"ds_sub_u64", VReg_64>;
519defm DS_RSUB_U64      : DS_1A1D_NORET_mc<"ds_rsub_u64", VReg_64>;
520defm DS_INC_U64       : DS_1A1D_NORET_mc<"ds_inc_u64", VReg_64>;
521defm DS_DEC_U64       : DS_1A1D_NORET_mc<"ds_dec_u64", VReg_64>;
522defm DS_MIN_I64       : DS_1A1D_NORET_mc<"ds_min_i64", VReg_64>;
523defm DS_MAX_I64       : DS_1A1D_NORET_mc<"ds_max_i64", VReg_64>;
524defm DS_MIN_U64       : DS_1A1D_NORET_mc<"ds_min_u64", VReg_64>;
525defm DS_MAX_U64       : DS_1A1D_NORET_mc<"ds_max_u64", VReg_64>;
526defm DS_AND_B64       : DS_1A1D_NORET_mc<"ds_and_b64", VReg_64>;
527defm DS_OR_B64        : DS_1A1D_NORET_mc<"ds_or_b64", VReg_64>;
528defm DS_XOR_B64       : DS_1A1D_NORET_mc<"ds_xor_b64", VReg_64>;
529defm DS_MSKOR_B64     : DS_1A2D_NORET_mc<"ds_mskor_b64", VReg_64>;
530let mayLoad = 0 in {
531defm DS_WRITE_B64     : DS_1A1D_NORET_mc<"ds_write_b64", VReg_64>;
532defm DS_WRITE2_B64    : DS_1A2D_Off8_NORET_mc<"ds_write2_b64", VReg_64>;
533defm DS_WRITE2ST64_B64: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b64", VReg_64>;
534}
535defm DS_CMPST_B64     : DS_1A2D_NORET_mc<"ds_cmpst_b64", VReg_64>;
536defm DS_CMPST_F64     : DS_1A2D_NORET_mc<"ds_cmpst_f64", VReg_64>;
537defm DS_MIN_F64       : DS_1A1D_NORET_mc<"ds_min_f64", VReg_64>;
538defm DS_MAX_F64       : DS_1A1D_NORET_mc<"ds_max_f64", VReg_64>;
539
540defm DS_ADD_RTN_U32   : DS_1A1D_RET_mc<"ds_add_rtn_u32", VGPR_32, "ds_add_u32">;
541
542let SubtargetPredicate = HasLDSFPAtomicAdd in {
543defm DS_ADD_RTN_F32   : DS_1A1D_RET_mc<"ds_add_rtn_f32", VGPR_32, "ds_add_f32">;
544}
545defm DS_SUB_RTN_U32   : DS_1A1D_RET_mc<"ds_sub_rtn_u32", VGPR_32, "ds_sub_u32">;
546defm DS_RSUB_RTN_U32  : DS_1A1D_RET_mc<"ds_rsub_rtn_u32", VGPR_32, "ds_rsub_u32">;
547defm DS_INC_RTN_U32   : DS_1A1D_RET_mc<"ds_inc_rtn_u32", VGPR_32, "ds_inc_u32">;
548defm DS_DEC_RTN_U32   : DS_1A1D_RET_mc<"ds_dec_rtn_u32", VGPR_32, "ds_dec_u32">;
549defm DS_MIN_RTN_I32   : DS_1A1D_RET_mc<"ds_min_rtn_i32", VGPR_32, "ds_min_i32">;
550defm DS_MAX_RTN_I32   : DS_1A1D_RET_mc<"ds_max_rtn_i32", VGPR_32, "ds_max_i32">;
551defm DS_MIN_RTN_U32   : DS_1A1D_RET_mc<"ds_min_rtn_u32", VGPR_32, "ds_min_u32">;
552defm DS_MAX_RTN_U32   : DS_1A1D_RET_mc<"ds_max_rtn_u32", VGPR_32, "ds_max_u32">;
553defm DS_AND_RTN_B32   : DS_1A1D_RET_mc<"ds_and_rtn_b32", VGPR_32, "ds_and_b32">;
554defm DS_OR_RTN_B32    : DS_1A1D_RET_mc<"ds_or_rtn_b32", VGPR_32, "ds_or_b32">;
555defm DS_XOR_RTN_B32   : DS_1A1D_RET_mc<"ds_xor_rtn_b32", VGPR_32, "ds_xor_b32">;
556defm DS_MSKOR_RTN_B32 : DS_1A2D_RET_mc<"ds_mskor_rtn_b32", VGPR_32, "ds_mskor_b32">;
557defm DS_CMPST_RTN_B32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_b32", VGPR_32, "ds_cmpst_b32">;
558defm DS_CMPST_RTN_F32 : DS_1A2D_RET_mc<"ds_cmpst_rtn_f32", VGPR_32, "ds_cmpst_f32">;
559defm DS_MIN_RTN_F32   : DS_1A1D_RET_mc<"ds_min_rtn_f32", VGPR_32, "ds_min_f32">;
560defm DS_MAX_RTN_F32   : DS_1A1D_RET_mc<"ds_max_rtn_f32", VGPR_32, "ds_max_f32">;
561
562defm DS_WRXCHG_RTN_B32 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b32">;
563defm DS_WRXCHG2_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b32", VReg_64, VGPR_32>;
564defm DS_WRXCHG2ST64_RTN_B32 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b32", VReg_64, VGPR_32>;
565
566defm DS_ADD_RTN_U64  : DS_1A1D_RET_mc<"ds_add_rtn_u64", VReg_64, "ds_add_u64">;
567defm DS_SUB_RTN_U64  : DS_1A1D_RET_mc<"ds_sub_rtn_u64", VReg_64, "ds_sub_u64">;
568defm DS_RSUB_RTN_U64  : DS_1A1D_RET_mc<"ds_rsub_rtn_u64", VReg_64, "ds_rsub_u64">;
569defm DS_INC_RTN_U64   : DS_1A1D_RET_mc<"ds_inc_rtn_u64", VReg_64, "ds_inc_u64">;
570defm DS_DEC_RTN_U64   : DS_1A1D_RET_mc<"ds_dec_rtn_u64", VReg_64, "ds_dec_u64">;
571defm DS_MIN_RTN_I64    : DS_1A1D_RET_mc<"ds_min_rtn_i64", VReg_64, "ds_min_i64">;
572defm DS_MAX_RTN_I64    : DS_1A1D_RET_mc<"ds_max_rtn_i64", VReg_64, "ds_max_i64">;
573defm DS_MIN_RTN_U64   : DS_1A1D_RET_mc<"ds_min_rtn_u64", VReg_64, "ds_min_u64">;
574defm DS_MAX_RTN_U64   : DS_1A1D_RET_mc<"ds_max_rtn_u64", VReg_64, "ds_max_u64">;
575defm DS_AND_RTN_B64    : DS_1A1D_RET_mc<"ds_and_rtn_b64", VReg_64, "ds_and_b64">;
576defm DS_OR_RTN_B64     : DS_1A1D_RET_mc<"ds_or_rtn_b64", VReg_64, "ds_or_b64">;
577defm DS_XOR_RTN_B64    : DS_1A1D_RET_mc<"ds_xor_rtn_b64", VReg_64, "ds_xor_b64">;
578defm DS_MSKOR_RTN_B64  : DS_1A2D_RET_mc<"ds_mskor_rtn_b64", VReg_64, "ds_mskor_b64">;
579defm DS_CMPST_RTN_B64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_b64", VReg_64, "ds_cmpst_b64">;
580defm DS_CMPST_RTN_F64  : DS_1A2D_RET_mc<"ds_cmpst_rtn_f64", VReg_64, "ds_cmpst_f64">;
581defm DS_MIN_RTN_F64    : DS_1A1D_RET_mc<"ds_min_rtn_f64", VReg_64, "ds_min_f64">;
582defm DS_MAX_RTN_F64    : DS_1A1D_RET_mc<"ds_max_rtn_f64", VReg_64, "ds_max_f64">;
583
584defm DS_WRXCHG_RTN_B64 : DS_1A1D_RET_mc<"ds_wrxchg_rtn_b64", VReg_64>;
585defm DS_WRXCHG2_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2_rtn_b64", VReg_128, VReg_64>;
586defm DS_WRXCHG2ST64_RTN_B64 : DS_1A2D_Off8_RET_mc<"ds_wrxchg2st64_rtn_b64", VReg_128, VReg_64>;
587
588let isConvergent = 1, usesCustomInserter = 1 in {
589def DS_GWS_INIT       : DS_GWS_1D<"ds_gws_init"> {
590  let mayLoad = 0;
591}
592def DS_GWS_SEMA_V     : DS_GWS_0D<"ds_gws_sema_v">;
593def DS_GWS_SEMA_BR    : DS_GWS_1D<"ds_gws_sema_br">;
594def DS_GWS_SEMA_P     : DS_GWS_0D<"ds_gws_sema_p">;
595def DS_GWS_BARRIER    : DS_GWS_1D<"ds_gws_barrier">;
596}
597
598let SubtargetPredicate = HasDsSrc2Insts in {
599def DS_ADD_SRC2_U32   : DS_1A<"ds_add_src2_u32">;
600def DS_SUB_SRC2_U32   : DS_1A<"ds_sub_src2_u32">;
601def DS_RSUB_SRC2_U32  : DS_1A<"ds_rsub_src2_u32">;
602def DS_INC_SRC2_U32   : DS_1A<"ds_inc_src2_u32">;
603def DS_DEC_SRC2_U32   : DS_1A<"ds_dec_src2_u32">;
604def DS_MIN_SRC2_I32   : DS_1A<"ds_min_src2_i32">;
605def DS_MAX_SRC2_I32   : DS_1A<"ds_max_src2_i32">;
606def DS_MIN_SRC2_U32   : DS_1A<"ds_min_src2_u32">;
607def DS_MAX_SRC2_U32   : DS_1A<"ds_max_src2_u32">;
608def DS_AND_SRC2_B32   : DS_1A<"ds_and_src2_b32">;
609def DS_OR_SRC2_B32    : DS_1A<"ds_or_src2_b32">;
610def DS_XOR_SRC2_B32   : DS_1A<"ds_xor_src2_b32">;
611def DS_MIN_SRC2_F32   : DS_1A<"ds_min_src2_f32">;
612def DS_MAX_SRC2_F32   : DS_1A<"ds_max_src2_f32">;
613
614def DS_ADD_SRC2_U64   : DS_1A<"ds_add_src2_u64">;
615def DS_SUB_SRC2_U64   : DS_1A<"ds_sub_src2_u64">;
616def DS_RSUB_SRC2_U64  : DS_1A<"ds_rsub_src2_u64">;
617def DS_INC_SRC2_U64   : DS_1A<"ds_inc_src2_u64">;
618def DS_DEC_SRC2_U64   : DS_1A<"ds_dec_src2_u64">;
619def DS_MIN_SRC2_I64   : DS_1A<"ds_min_src2_i64">;
620def DS_MAX_SRC2_I64   : DS_1A<"ds_max_src2_i64">;
621def DS_MIN_SRC2_U64   : DS_1A<"ds_min_src2_u64">;
622def DS_MAX_SRC2_U64   : DS_1A<"ds_max_src2_u64">;
623def DS_AND_SRC2_B64   : DS_1A<"ds_and_src2_b64">;
624def DS_OR_SRC2_B64    : DS_1A<"ds_or_src2_b64">;
625def DS_XOR_SRC2_B64   : DS_1A<"ds_xor_src2_b64">;
626def DS_MIN_SRC2_F64   : DS_1A<"ds_min_src2_f64">;
627def DS_MAX_SRC2_F64   : DS_1A<"ds_max_src2_f64">;
628
629def DS_WRITE_SRC2_B32 : DS_1A<"ds_write_src2_b32">;
630def DS_WRITE_SRC2_B64 : DS_1A<"ds_write_src2_b64">;
631} // End SubtargetPredicate = HasDsSrc2Insts
632
633let Uses = [EXEC], mayLoad = 0, mayStore = 0, isConvergent = 1 in {
634def DS_SWIZZLE_B32 : DS_1A_RET <"ds_swizzle_b32", VGPR_32, 0, SwizzleImm>;
635}
636
637let mayStore = 0 in {
638defm DS_READ_I8      : DS_1A_RET_mc<"ds_read_i8">;
639defm DS_READ_U8      : DS_1A_RET_mc<"ds_read_u8">;
640defm DS_READ_I16     : DS_1A_RET_mc<"ds_read_i16">;
641defm DS_READ_U16     : DS_1A_RET_mc<"ds_read_u16">;
642defm DS_READ_B32     : DS_1A_RET_mc<"ds_read_b32">;
643defm DS_READ_B64     : DS_1A_RET_mc<"ds_read_b64", VReg_64>;
644
645defm DS_READ2_B32    : DS_1A_Off8_RET_mc<"ds_read2_b32", VReg_64>;
646defm DS_READ2ST64_B32: DS_1A_Off8_RET_mc<"ds_read2st64_b32", VReg_64>;
647
648defm DS_READ2_B64    : DS_1A_Off8_RET_mc<"ds_read2_b64", VReg_128>;
649defm DS_READ2ST64_B64: DS_1A_Off8_RET_mc<"ds_read2st64_b64", VReg_128>;
650
651let has_m0_read = 0 in {
652let SubtargetPredicate = HasD16LoadStore, TiedSourceNotRead = 1 in {
653def DS_READ_U8_D16     : DS_1A_RET_Tied<"ds_read_u8_d16">;
654def DS_READ_U8_D16_HI  : DS_1A_RET_Tied<"ds_read_u8_d16_hi">;
655def DS_READ_I8_D16     : DS_1A_RET_Tied<"ds_read_i8_d16">;
656def DS_READ_I8_D16_HI  : DS_1A_RET_Tied<"ds_read_i8_d16_hi">;
657def DS_READ_U16_D16    : DS_1A_RET_Tied<"ds_read_u16_d16">;
658def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">;
659}
660} // End has_m0_read = 0
661
662let SubtargetPredicate = HasDSAddTid in {
663def DS_READ_ADDTID_B32 : DS_0A_RET<"ds_read_addtid_b32">;
664}
665
666} // End mayStore = 0
667
668def DS_CONSUME       : DS_0A_RET<"ds_consume">;
669def DS_APPEND        : DS_0A_RET<"ds_append">;
670
671let SubtargetPredicate = isNotGFX90APlus in
672def DS_ORDERED_COUNT : DS_1A_RET_GDS<"ds_ordered_count">;
673
674//===----------------------------------------------------------------------===//
675// Instruction definitions for CI and newer.
676//===----------------------------------------------------------------------===//
677
678let SubtargetPredicate = isGFX7Plus in {
679
680defm DS_WRAP_RTN_B32 : DS_1A2D_RET_mc<"ds_wrap_rtn_b32", VGPR_32>;
681defm DS_CONDXCHG32_RTN_B64 : DS_1A1D_RET_mc<"ds_condxchg32_rtn_b64", VReg_64>;
682
683let isConvergent = 1, usesCustomInserter = 1 in {
684def DS_GWS_SEMA_RELEASE_ALL : DS_GWS_0D<"ds_gws_sema_release_all">;
685}
686
687let mayStore = 0 in {
688defm DS_READ_B96 : DS_1A_RET_mc<"ds_read_b96", VReg_96>;
689defm DS_READ_B128: DS_1A_RET_mc<"ds_read_b128", VReg_128>;
690} // End mayStore = 0
691
692let mayLoad = 0 in {
693defm DS_WRITE_B96 : DS_1A1D_NORET_mc<"ds_write_b96", VReg_96>;
694defm DS_WRITE_B128 : DS_1A1D_NORET_mc<"ds_write_b128", VReg_128>;
695} // End mayLoad = 0
696
697def DS_NOP : DS_VOID<"ds_nop">;
698
699} // let SubtargetPredicate = isGFX7Plus
700
701//===----------------------------------------------------------------------===//
702// Instruction definitions for VI and newer.
703//===----------------------------------------------------------------------===//
704
705let SubtargetPredicate = isGFX8Plus in {
706
707let Uses = [EXEC] in {
708def DS_PERMUTE_B32  : DS_1A1D_PERMUTE <"ds_permute_b32",
709                                       int_amdgcn_ds_permute>;
710def DS_BPERMUTE_B32 : DS_1A1D_PERMUTE <"ds_bpermute_b32",
711                                       int_amdgcn_ds_bpermute>;
712}
713
714} // let SubtargetPredicate = isGFX8Plus
715
716let SubtargetPredicate = HasLDSFPAtomicAdd, OtherPredicates = [HasDsSrc2Insts] in {
717def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;
718}
719
720
721//===----------------------------------------------------------------------===//
722// Instruction definitions for GFX11 and newer.
723//===----------------------------------------------------------------------===//
724
725let SubtargetPredicate = isGFX11Plus in {
726
727def DS_ADD_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_add_gs_reg_rtn", VReg_64, VGPR_32>;
728def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VReg_64, VGPR_32>;
729def DS_BVH_STACK_RTN_B32 : DS_BVH_STACK<"ds_bvh_stack_rtn_b32">;
730
731} // let SubtargetPredicate = isGFX11Plus
732
733//===----------------------------------------------------------------------===//
734// DS Patterns
735//===----------------------------------------------------------------------===//
736
737def : GCNPat <
738  (int_amdgcn_ds_swizzle i32:$src, timm:$offset16),
739  (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0))
740>;
741
742class DSReadPat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
743  (vt (frag (DS1Addr1Offset i32:$ptr, i16:$offset))),
744  (inst $ptr, offset:$offset, (i1 gds))
745>;
746
747multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
748
749  let OtherPredicates = [LDSRequiresM0Init] in {
750    def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
751  }
752
753  let OtherPredicates = [NotLDSRequiresM0Init] in {
754    def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
755  }
756}
757
758class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <
759  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$in),
760  (inst $ptr, offset:$offset, (i1 0), $in)
761>;
762
763defm : DSReadPat_mc <DS_READ_I8, i32, "sextloadi8_local">;
764defm : DSReadPat_mc <DS_READ_I8,  i16, "sextloadi8_local">;
765defm : DSReadPat_mc <DS_READ_U8,  i32, "extloadi8_local">;
766defm : DSReadPat_mc <DS_READ_U8,  i32, "zextloadi8_local">;
767defm : DSReadPat_mc <DS_READ_U8,  i16, "extloadi8_local">;
768defm : DSReadPat_mc <DS_READ_U8,  i16, "zextloadi8_local">;
769defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
770defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
771defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
772defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
773defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
774
775foreach vt = Reg32Types.types in {
776defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
777}
778
779defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_8_local">;
780defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_8_local">;
781defm : DSReadPat_mc <DS_READ_U16, i16, "atomic_load_16_local">;
782defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_16_local">;
783defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
784defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
785
786let OtherPredicates = [D16PreservesUnusedBits] in {
787def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2i16>;
788def : DSReadPat_D16<DS_READ_U16_D16_HI, load_d16_hi_local, v2f16>;
789def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2i16>;
790def : DSReadPat_D16<DS_READ_U8_D16_HI, az_extloadi8_d16_hi_local, v2f16>;
791def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2i16>;
792def : DSReadPat_D16<DS_READ_I8_D16_HI, sextloadi8_d16_hi_local, v2f16>;
793
794def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2i16>;
795def : DSReadPat_D16<DS_READ_U16_D16, load_d16_lo_local, v2f16>;
796def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2i16>;
797def : DSReadPat_D16<DS_READ_U8_D16, az_extloadi8_d16_lo_local, v2f16>;
798def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2i16>;
799def : DSReadPat_D16<DS_READ_I8_D16, sextloadi8_d16_lo_local, v2f16>;
800}
801
802class DSWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat <
803  (frag vt:$value, (DS1Addr1Offset i32:$ptr, i16:$offset)),
804  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))
805>;
806
807multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
808  let OtherPredicates = [LDSRequiresM0Init] in {
809    def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
810  }
811
812  let OtherPredicates = [NotLDSRequiresM0Init] in {
813    def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
814  }
815}
816
817// Irritatingly, atomic_store reverses the order of operands from a
818// normal store.
819class DSAtomicWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
820  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
821  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 0))
822>;
823
824multiclass DSAtomicWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
825  let OtherPredicates = [LDSRequiresM0Init] in {
826    def : DSAtomicWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
827  }
828
829  let OtherPredicates = [NotLDSRequiresM0Init] in {
830    def : DSAtomicWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
831  }
832}
833
834defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
835defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
836defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
837defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
838
839foreach vt = Reg32Types.types in {
840defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
841}
842
843defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
844defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
845defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
846defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
847defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
848defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
849
850let OtherPredicates = [HasD16LoadStore] in {
851def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;
852def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>;
853}
854
855class DS64Bit4ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
856  (vt:$value (frag (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
857  (inst $ptr, $offset0, $offset1, (i1 0))
858>;
859
860class DS64Bit4ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
861  (frag vt:$value, (DS64Bit4ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
862  (inst $ptr, (i32 (EXTRACT_SUBREG VReg_64:$value, sub0)),
863              (i32 (EXTRACT_SUBREG VReg_64:$value, sub1)), $offset0, $offset1,
864              (i1 0))
865>;
866
867class DS128Bit8ByteAlignedReadPat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
868  (vt:$value (frag (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1))),
869  (inst $ptr, $offset0, $offset1, (i1 0))
870>;
871
872class DS128Bit8ByteAlignedWritePat<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat<
873  (frag vt:$value, (DS128Bit8ByteAligned i32:$ptr, i8:$offset0, i8:$offset1)),
874  (inst $ptr, (i64 (EXTRACT_SUBREG VReg_128:$value, sub0_sub1)),
875              (i64 (EXTRACT_SUBREG VReg_128:$value, sub2_sub3)), $offset0, $offset1,
876              (i1 0))
877>;
878
879multiclass DS64Bit4ByteAlignedPat_mc<ValueType vt> {
880  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
881    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32, vt, load_local_m0>;
882    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32, vt, store_local_m0>;
883  }
884
885  let OtherPredicates = [NotLDSRequiresM0Init] in {
886    def : DS64Bit4ByteAlignedReadPat<DS_READ2_B32_gfx9, vt, load_local>;
887    def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, vt, store_local>;
888  }
889}
890
891multiclass DS128Bit8ByteAlignedPat_mc<ValueType vt> {
892  let OtherPredicates = [LDSRequiresM0Init, isGFX7Plus] in {
893    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64, vt, load_local_m0>;
894    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64, vt, store_local_m0>;
895  }
896
897  let OtherPredicates = [NotLDSRequiresM0Init] in {
898    def : DS128Bit8ByteAlignedReadPat<DS_READ2_B64_gfx9, vt, load_local>;
899    def : DS128Bit8ByteAlignedWritePat<DS_WRITE2_B64_gfx9, vt, store_local>;
900  }
901}
902
903// v2i32 loads are split into i32 loads on SI during lowering, due to a bug
904// related to bounds checking.
905foreach vt = VReg_64.RegTypes in {
906defm : DS64Bit4ByteAlignedPat_mc<vt>;
907}
908
909foreach vt = VReg_128.RegTypes in {
910defm : DS128Bit8ByteAlignedPat_mc<vt>;
911}
912
913// Prefer ds_read over ds_read2 and ds_write over ds_write2, all other things
914// being equal, because it has a larger immediate offset range.
915let AddedComplexity = 100 in {
916
917foreach vt = VReg_64.RegTypes in {
918defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
919defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
920}
921
922let SubtargetPredicate = isGFX7Plus in {
923
924foreach vt = VReg_96.RegTypes in {
925defm : DSReadPat_mc <DS_READ_B96, vt, "load_align16_local">;
926defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_align16_local">;
927}
928
929foreach vt = VReg_128.RegTypes in {
930defm : DSReadPat_mc <DS_READ_B128, vt, "load_align16_local">;
931defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align16_local">;
932}
933
934let SubtargetPredicate = HasUnalignedAccessMode in {
935
936// Select 64 bit loads and stores aligned less than 4 as a single ds_read_b64/
937// ds_write_b64 instruction as this is faster than ds_read2_b32/ds_write2_b32
938// which would be used otherwise. In this case a b32 access would still be
939// misaligned, but we will have 2 of them.
940foreach vt = VReg_64.RegTypes in {
941defm : DSReadPat_mc <DS_READ_B64, vt, "load_align_less_than_4_local">;
942defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align_less_than_4_local">;
943}
944
945// Selection will split most of the unaligned 3 dword accesses due to performance
946// reasons when beneficial. Keep these two patterns for the rest of the cases.
947foreach vt = VReg_96.RegTypes in {
948defm : DSReadPat_mc <DS_READ_B96, vt, "load_local">;
949defm : DSWritePat_mc <DS_WRITE_B96, vt, "store_local">;
950}
951
952// Select 128 bit loads and stores aligned less than 4 as a single ds_read_b128/
953// ds_write_b128 instruction as this is faster than ds_read2_b64/ds_write2_b64
954// which would be used otherwise. In this case a b64 access would still be
955// misaligned, but we will have 2 of them.
956foreach vt = VReg_128.RegTypes in {
957defm : DSReadPat_mc <DS_READ_B128, vt, "load_align_less_than_4_local">;
958defm : DSWritePat_mc <DS_WRITE_B128, vt, "store_align_less_than_4_local">;
959}
960
961} // End SubtargetPredicate = HasUnalignedAccessMode
962
963} // End SubtargetPredicate = isGFX7Plus
964
965} // End AddedComplexity = 100
966
967class DSAtomicRetPat<DS_Pseudo inst, ValueType vt, PatFrag frag, int complexity = 0,
968  bit gds=0> : GCNPat <(frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value),
969  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))> {
970  let AddedComplexity = complexity;
971}
972
973multiclass DSAtomicRetPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
974  let OtherPredicates = [LDSRequiresM0Init] in {
975    def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
976  }
977
978  let OtherPredicates = [NotLDSRequiresM0Init] in {
979    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
980                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
981  }
982
983  def : DSAtomicRetPat<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
984                       /* complexity */ 0, /* gds */ 1>;
985}
986
987multiclass DSAtomicRetNoRetPat_mc<DS_Pseudo inst, DS_Pseudo noRetInst,
988                                  ValueType vt, string frag> {
989  let OtherPredicates = [LDSRequiresM0Init] in {
990    def : DSAtomicRetPat<inst, vt,
991                         !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
992    def : DSAtomicRetPat<noRetInst, vt,
993                         !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size), /* complexity */ 1>;
994  }
995
996  let OtherPredicates = [NotLDSRequiresM0Init] in {
997    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
998                         !cast<PatFrag>(frag#"_local_"#vt.Size)>;
999    def : DSAtomicRetPat<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1000                         !cast<PatFrag>(frag#"_local_noret_"#vt.Size), /* complexity */ 1>;
1001  }
1002
1003  def : DSAtomicRetPat<inst, vt,
1004                       !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1005                       /* complexity */ 0, /* gds */ 1>;
1006  def : DSAtomicRetPat<noRetInst, vt,
1007                       !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1008                       /* complexity */ 1, /* gds */ 1>;
1009}
1010
1011
1012
1013let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1014// Caution, the order of src and cmp is the *opposite* of the BUFFER_ATOMIC_CMPSWAP opcode.
1015class DSAtomicCmpXChgSwapped<DS_Pseudo inst, ValueType vt, PatFrag frag,
1016  int complexity = 0, bit gds=0> : GCNPat<
1017  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap),
1018  (inst $ptr, getVregSrcForVT<vt>.ret:$cmp, getVregSrcForVT<vt>.ret:$swap, offset:$offset, (i1 gds))> {
1019  let AddedComplexity = complexity;
1020}
1021
1022multiclass DSAtomicCmpXChgSwapped_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt,
1023                                     string frag> {
1024  let OtherPredicates = [LDSRequiresM0Init] in {
1025    def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_local_m0_"#vt.Size)>;
1026    def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_local_m0_noret_"#vt.Size),
1027                                 /* complexity */ 1>;
1028  }
1029
1030  let OtherPredicates = [NotLDSRequiresM0Init] in {
1031    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1032                                 !cast<PatFrag>(frag#"_local_"#vt.Size)>;
1033    def : DSAtomicCmpXChgSwapped<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1034                                 !cast<PatFrag>(frag#"_local_noret_"#vt.Size),
1035                                 /* complexity */ 1>;
1036  }
1037
1038  def : DSAtomicCmpXChgSwapped<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1039                               /* complexity */ 0, /* gds */ 1>;
1040  def : DSAtomicCmpXChgSwapped<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1041                               /* complexity */ 1, /* gds */ 1>;
1042}
1043} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1044
1045let SubtargetPredicate = isGFX11Plus in {
1046// The order of src and cmp agrees with the BUFFER_ATOMIC_CMPSWAP opcode.
1047class DSAtomicCmpXChg<DS_Pseudo inst, ValueType vt, PatFrag frag,
1048  int complexity = 0, bit gds=0> : GCNPat<
1049  (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap),
1050  (inst $ptr, getVregSrcForVT<vt>.ret:$swap, getVregSrcForVT<vt>.ret:$cmp, offset:$offset, (i1 gds))> {
1051  let AddedComplexity = complexity;
1052}
1053
1054multiclass DSAtomicCmpXChg_mc<DS_Pseudo inst, DS_Pseudo noRetInst, ValueType vt, string frag> {
1055
1056  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt,
1057                        !cast<PatFrag>(frag#"_local_"#vt.Size)>;
1058  def : DSAtomicCmpXChg<!cast<DS_Pseudo>(!cast<string>(noRetInst)#"_gfx9"), vt,
1059                        !cast<PatFrag>(frag#"_local_noret_"#vt.Size), /* complexity */ 1>;
1060
1061  def : DSAtomicCmpXChg<inst, vt, !cast<PatFrag>(frag#"_region_m0_"#vt.Size),
1062                        /* complexity */ 0, /* gds */ 1>;
1063  def : DSAtomicCmpXChg<noRetInst, vt, !cast<PatFrag>(frag#"_region_m0_noret_"#vt.Size),
1064                        /* complexity */ 1, /* gds */ 1>;
1065}
1066} // End SubtargetPredicate = isGFX11Plus
1067
1068// 32-bit atomics.
1069defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B32, i32, "atomic_swap">;
1070defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U32, DS_ADD_U32, i32, "atomic_load_add">;
1071defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U32, DS_SUB_U32, i32, "atomic_load_sub">;
1072defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U32, DS_INC_U32, i32, "atomic_inc">;
1073defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U32, DS_DEC_U32, i32, "atomic_dec">;
1074defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B32, DS_AND_B32, i32, "atomic_load_and">;
1075defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B32, DS_OR_B32, i32, "atomic_load_or">;
1076defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B32, DS_XOR_B32, i32, "atomic_load_xor">;
1077defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I32, DS_MIN_I32, i32, "atomic_load_min">;
1078defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I32, DS_MAX_I32, i32, "atomic_load_max">;
1079defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U32, DS_MIN_U32, i32, "atomic_load_umin">;
1080defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U32, DS_MAX_U32, i32, "atomic_load_umax">;
1081defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F32, DS_MIN_F32, f32, "atomic_load_fmin">;
1082defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F32, DS_MAX_F32, f32, "atomic_load_fmax">;
1083
1084let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1085defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B32, DS_CMPST_B32, i32, "atomic_cmp_swap">;
1086}
1087
1088let SubtargetPredicate = isGFX11Plus in {
1089defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B32, DS_CMPSTORE_B32, i32, "atomic_cmp_swap">;
1090}
1091
1092let SubtargetPredicate = HasLDSFPAtomicAdd in {
1093defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_F32, DS_ADD_F32, f32, "atomic_load_fadd">;
1094}
1095
1096// 64-bit atomics.
1097defm : DSAtomicRetPat_mc<DS_WRXCHG_RTN_B64, i64, "atomic_swap">;
1098defm : DSAtomicRetNoRetPat_mc<DS_ADD_RTN_U64, DS_ADD_U64, i64, "atomic_load_add">;
1099defm : DSAtomicRetNoRetPat_mc<DS_SUB_RTN_U64, DS_SUB_U64, i64, "atomic_load_sub">;
1100defm : DSAtomicRetNoRetPat_mc<DS_INC_RTN_U64, DS_INC_U64, i64, "atomic_inc">;
1101defm : DSAtomicRetNoRetPat_mc<DS_DEC_RTN_U64, DS_DEC_U64, i64, "atomic_dec">;
1102defm : DSAtomicRetNoRetPat_mc<DS_AND_RTN_B64, DS_AND_B64, i64, "atomic_load_and">;
1103defm : DSAtomicRetNoRetPat_mc<DS_OR_RTN_B64, DS_OR_B64, i64, "atomic_load_or">;
1104defm : DSAtomicRetNoRetPat_mc<DS_XOR_RTN_B64, DS_XOR_B64, i64, "atomic_load_xor">;
1105defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_I64, DS_MIN_I64, i64, "atomic_load_min">;
1106defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_I64, DS_MAX_I64, i64, "atomic_load_max">;
1107defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_U64, DS_MIN_U64, i64, "atomic_load_umin">;
1108defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_U64, DS_MAX_U64, i64, "atomic_load_umax">;
1109defm : DSAtomicRetNoRetPat_mc<DS_MIN_RTN_F64, DS_MIN_F64, f64, "atomic_load_fmin">;
1110defm : DSAtomicRetNoRetPat_mc<DS_MAX_RTN_F64, DS_MAX_F64, f64, "atomic_load_fmax">;
1111
1112let SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10 in {
1113defm : DSAtomicCmpXChgSwapped_mc<DS_CMPST_RTN_B64, DS_CMPST_B64, i64, "atomic_cmp_swap">;
1114} // End SubtargetPredicate = isGFX6GFX7GFX8GFX9GFX10
1115
1116let SubtargetPredicate = isGFX11Plus in {
1117defm : DSAtomicCmpXChg_mc<DS_CMPSTORE_RTN_B64, DS_CMPSTORE_B64, i64, "atomic_cmp_swap">;
1118} // End SubtargetPredicate = isGFX11Plus
1119
1120let SubtargetPredicate = isGFX90APlus in {
1121def : DSAtomicRetPat<DS_ADD_RTN_F64, f64, atomic_load_fadd_local_64>;
1122let AddedComplexity = 1 in
1123def : DSAtomicRetPat<DS_ADD_F64, f64, atomic_load_fadd_local_noret_64>;
1124
1125class DSAtomicRetPatIntrinsic<DS_Pseudo inst, ValueType vt, PatFrag frag,
1126  bit gds=0> : GCNPat <
1127  (vt (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value)),
1128  (inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 gds))> {
1129}
1130
1131def : DSAtomicRetPatIntrinsic<DS_ADD_RTN_F64, f64, int_amdgcn_flat_atomic_fadd_local_addrspace>;
1132let AddedComplexity = 1 in
1133def : DSAtomicRetPatIntrinsic<DS_ADD_F64, f64, int_amdgcn_flat_atomic_fadd_noret_local_addrspace>;
1134}
1135
1136let SubtargetPredicate = isGFX940Plus in {
1137def : DSAtomicRetPat<DS_PK_ADD_RTN_F16, v2f16, atomic_load_fadd_v2f16_local_32>;
1138let AddedComplexity = 1 in
1139def : DSAtomicRetPat<DS_PK_ADD_F16, v2f16, atomic_load_fadd_v2f16_local_noret_32>;
1140def : GCNPat <
1141  (v2i16 (int_amdgcn_ds_fadd_v2bf16 i32:$ptr, v2i16:$src)),
1142  (DS_PK_ADD_RTN_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1143>;
1144let AddedComplexity = 1 in
1145def : GCNPat <
1146  (v2i16 (int_amdgcn_ds_fadd_v2bf16_noret i32:$ptr, v2i16:$src)),
1147  (DS_PK_ADD_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0)
1148>;
1149}
1150
1151def : Pat <
1152  (SIds_ordered_count i32:$value, i16:$offset),
1153  (DS_ORDERED_COUNT $value, (as_i16imm $offset))
1154>;
1155
1156def : GCNPat <
1157  (i64 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),
1158  (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))
1159>;
1160
1161def : GCNPat <
1162  (i32 (int_amdgcn_ds_add_gs_reg_rtn i32:$src, timm:$offset32)),
1163  (EXTRACT_SUBREG
1164    (i64 (COPY_TO_REGCLASS
1165      (DS_ADD_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),
1166      VReg_64)),
1167    sub0)
1168>;
1169
1170def : GCNPat <
1171  (i64 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),
1172  (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32))
1173>;
1174
1175def : GCNPat <
1176  (i32 (int_amdgcn_ds_sub_gs_reg_rtn i32:$src, timm:$offset32)),
1177  (EXTRACT_SUBREG
1178    (i64 (COPY_TO_REGCLASS
1179      (DS_SUB_GS_REG_RTN VGPR_32:$src, (as_i32timm $offset32)),
1180      VReg_64)),
1181    sub0)
1182>;
1183
1184//===----------------------------------------------------------------------===//
1185// Target-specific instruction encodings.
1186//===----------------------------------------------------------------------===//
1187
1188//===----------------------------------------------------------------------===//
1189// Base ENC_DS for GFX6, GFX7, GFX10, GFX11.
1190//===----------------------------------------------------------------------===//
1191
1192class Base_DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op, DS_Pseudo ps, int ef, string opName = ps.Mnemonic> :
1193    DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> {
1194
1195  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1196  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1197  let Inst{17}    = !if(ps.has_gds, gds, ps.gdsValue);
1198  let Inst{25-18} = op;
1199  let Inst{31-26} = 0x36;
1200  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1201  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1202  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1203  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1204}
1205
1206//===----------------------------------------------------------------------===//
1207// GFX11.
1208//===----------------------------------------------------------------------===//
1209
1210let AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11" in {
1211  multiclass DS_Real_gfx11<bits<8> op>  {
1212    def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1213                                              SIEncodingFamily.GFX11>;
1214  }
1215
1216  multiclass DS_Real_Renamed_gfx11<bits<8> op, DS_Pseudo backing_pseudo, string real_name> {
1217     def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, backing_pseudo, SIEncodingFamily.GFX11, real_name>,
1218               MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX11Plus]>;
1219  }
1220} // End AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11"
1221
1222defm DS_STORE_B32                        : DS_Real_Renamed_gfx11<0x00d, DS_WRITE_B32, "ds_store_b32">;
1223defm DS_STORE_2ADDR_B32                  : DS_Real_Renamed_gfx11<0x00e, DS_WRITE2_B32, "ds_store_2addr_b32">;
1224defm DS_STORE_2ADDR_STRIDE64_B32         : DS_Real_Renamed_gfx11<0x00f, DS_WRITE2ST64_B32, "ds_store_2addr_stride64_b32">;
1225defm DS_STORE_B8                         : DS_Real_Renamed_gfx11<0x01e, DS_WRITE_B8, "ds_store_b8">;
1226defm DS_STORE_B16                        : DS_Real_Renamed_gfx11<0x01f, DS_WRITE_B16, "ds_store_b16">;
1227defm DS_STOREXCHG_RTN_B32                : DS_Real_Renamed_gfx11<0x02d, DS_WRXCHG_RTN_B32, "ds_storexchg_rtn_b32">;
1228defm DS_STOREXCHG_2ADDR_RTN_B32          : DS_Real_Renamed_gfx11<0x02e, DS_WRXCHG2_RTN_B32, "ds_storexchg_2addr_rtn_b32">;
1229defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32 : DS_Real_Renamed_gfx11<0x02f, DS_WRXCHG2ST64_RTN_B32, "ds_storexchg_2addr_stride64_rtn_b32">;
1230defm DS_LOAD_B32                         : DS_Real_Renamed_gfx11<0x036, DS_READ_B32, "ds_load_b32">;
1231defm DS_LOAD_2ADDR_B32                   : DS_Real_Renamed_gfx11<0x037, DS_READ2_B32, "ds_load_2addr_b32">;
1232defm DS_LOAD_2ADDR_STRIDE64_B32          : DS_Real_Renamed_gfx11<0x038, DS_READ2ST64_B32, "ds_load_2addr_stride64_b32">;
1233defm DS_LOAD_I8                          : DS_Real_Renamed_gfx11<0x039, DS_READ_I8, "ds_load_i8">;
1234defm DS_LOAD_U8                          : DS_Real_Renamed_gfx11<0x03a, DS_READ_U8, "ds_load_u8">;
1235defm DS_LOAD_I16                         : DS_Real_Renamed_gfx11<0x03b, DS_READ_I16, "ds_load_i16">;
1236defm DS_LOAD_U16                         : DS_Real_Renamed_gfx11<0x03c, DS_READ_U16, "ds_load_u16">;
1237defm DS_STORE_B64                        : DS_Real_Renamed_gfx11<0x04d, DS_WRITE_B64, "ds_store_b64">;
1238defm DS_STORE_2ADDR_B64                  : DS_Real_Renamed_gfx11<0x04e, DS_WRITE2_B64, "ds_store_2addr_b64">;
1239defm DS_STORE_2ADDR_STRIDE64_B64         : DS_Real_Renamed_gfx11<0x04f, DS_WRITE2ST64_B64, "ds_store_2addr_stride64_b64">;
1240defm DS_STOREXCHG_RTN_B64                : DS_Real_Renamed_gfx11<0x06d, DS_WRXCHG_RTN_B64, "ds_storexchg_rtn_b64">;
1241defm DS_STOREXCHG_2ADDR_RTN_B64          : DS_Real_Renamed_gfx11<0x06e, DS_WRXCHG2_RTN_B64, "ds_storexchg_2addr_rtn_b64">;
1242defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64 : DS_Real_Renamed_gfx11<0x06f, DS_WRXCHG2ST64_RTN_B64, "ds_storexchg_2addr_stride64_rtn_b64">;
1243defm DS_LOAD_B64                         : DS_Real_Renamed_gfx11<0x076, DS_READ_B64, "ds_load_b64">;
1244defm DS_LOAD_2ADDR_B64                   : DS_Real_Renamed_gfx11<0x077, DS_READ2_B64, "ds_load_2addr_b64">;
1245defm DS_LOAD_2ADDR_STRIDE64_B64          : DS_Real_Renamed_gfx11<0x078, DS_READ2ST64_B64, "ds_load_2addr_stride64_b64">;
1246defm DS_STORE_B8_D16_HI                  : DS_Real_Renamed_gfx11<0x0a0, DS_WRITE_B8_D16_HI, "ds_store_b8_d16_hi">;
1247defm DS_STORE_B16_D16_HI                 : DS_Real_Renamed_gfx11<0x0a1, DS_WRITE_B16_D16_HI, "ds_store_b16_d16_hi">;
1248defm DS_LOAD_U8_D16                      : DS_Real_Renamed_gfx11<0x0a2, DS_READ_U8_D16, "ds_load_u8_d16">;
1249defm DS_LOAD_U8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a3, DS_READ_U8_D16_HI, "ds_load_u8_d16_hi">;
1250defm DS_LOAD_I8_D16                      : DS_Real_Renamed_gfx11<0x0a4, DS_READ_I8_D16, "ds_load_i8_d16">;
1251defm DS_LOAD_I8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a5, DS_READ_I8_D16_HI, "ds_load_i8_d16_hi">;
1252defm DS_LOAD_U16_D16                     : DS_Real_Renamed_gfx11<0x0a6, DS_READ_U16_D16, "ds_load_u16_d16">;
1253defm DS_LOAD_U16_D16_HI                  : DS_Real_Renamed_gfx11<0x0a7, DS_READ_U16_D16_HI, "ds_load_u16_d16_hi">;
1254defm DS_STORE_ADDTID_B32                 : DS_Real_Renamed_gfx11<0x0b0, DS_WRITE_ADDTID_B32, "ds_store_addtid_b32">;
1255defm DS_LOAD_ADDTID_B32                  : DS_Real_Renamed_gfx11<0x0b1, DS_READ_ADDTID_B32, "ds_load_addtid_b32">;
1256defm DS_STORE_B96                        : DS_Real_Renamed_gfx11<0x0de, DS_WRITE_B96, "ds_store_b96">;
1257defm DS_STORE_B128                       : DS_Real_Renamed_gfx11<0x0df, DS_WRITE_B128, "ds_store_b128">;
1258defm DS_LOAD_B96                         : DS_Real_Renamed_gfx11<0x0fe, DS_READ_B96, "ds_load_b96">;
1259defm DS_LOAD_B128                        : DS_Real_Renamed_gfx11<0x0ff, DS_READ_B128, "ds_load_b128">;
1260
1261// DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped
1262// comparing to pre-GFX11.
1263// Note: the mnemonic alias is not generated to avoid a potential ambiguity due to the semantics change.
1264
1265defm DS_CMPSTORE_B32                     : DS_Real_gfx11<0x010>;
1266defm DS_CMPSTORE_F32                     : DS_Real_gfx11<0x011>;
1267defm DS_CMPSTORE_RTN_B32                 : DS_Real_gfx11<0x030>;
1268defm DS_CMPSTORE_RTN_F32                 : DS_Real_gfx11<0x031>;
1269defm DS_CMPSTORE_B64                     : DS_Real_gfx11<0x050>;
1270defm DS_CMPSTORE_F64                     : DS_Real_gfx11<0x051>;
1271defm DS_CMPSTORE_RTN_B64                 : DS_Real_gfx11<0x070>;
1272defm DS_CMPSTORE_RTN_F64                 : DS_Real_gfx11<0x071>;
1273
1274defm DS_ADD_RTN_F32                      : DS_Real_gfx11<0x079>;
1275defm DS_ADD_GS_REG_RTN                   : DS_Real_gfx11<0x07a>;
1276defm DS_SUB_GS_REG_RTN                   : DS_Real_gfx11<0x07b>;
1277defm DS_BVH_STACK_RTN_B32                : DS_Real_gfx11<0x0ad>;
1278
1279//===----------------------------------------------------------------------===//
1280// GFX10.
1281//===----------------------------------------------------------------------===//
1282
1283let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
1284  multiclass DS_Real_gfx10<bits<8> op>  {
1285    def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1286                                              SIEncodingFamily.GFX10>;
1287  }
1288} // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
1289
1290defm DS_ADD_RTN_F32      : DS_Real_gfx10<0x055>;
1291defm DS_WRITE_B8_D16_HI  : DS_Real_gfx10<0x0a0>;
1292defm DS_WRITE_B16_D16_HI : DS_Real_gfx10<0x0a1>;
1293defm DS_READ_U8_D16      : DS_Real_gfx10<0x0a2>;
1294defm DS_READ_U8_D16_HI   : DS_Real_gfx10<0x0a3>;
1295defm DS_READ_I8_D16      : DS_Real_gfx10<0x0a4>;
1296defm DS_READ_I8_D16_HI   : DS_Real_gfx10<0x0a5>;
1297defm DS_READ_U16_D16     : DS_Real_gfx10<0x0a6>;
1298defm DS_READ_U16_D16_HI  : DS_Real_gfx10<0x0a7>;
1299defm DS_WRITE_ADDTID_B32 : DS_Real_gfx10<0x0b0>;
1300defm DS_READ_ADDTID_B32  : DS_Real_gfx10<0x0b1>;
1301
1302//===----------------------------------------------------------------------===//
1303// GFX10, GFX11.
1304//===----------------------------------------------------------------------===//
1305
1306multiclass DS_Real_gfx10_gfx11<bits<8> op> :
1307  DS_Real_gfx10<op>, DS_Real_gfx11<op>;
1308
1309defm DS_ADD_F32          : DS_Real_gfx10_gfx11<0x015>;
1310defm DS_ADD_SRC2_F32     : DS_Real_gfx10<0x095>;
1311defm DS_PERMUTE_B32      : DS_Real_gfx10_gfx11<0x0b2>;
1312defm DS_BPERMUTE_B32     : DS_Real_gfx10_gfx11<0x0b3>;
1313
1314//===----------------------------------------------------------------------===//
1315// GFX7, GFX10, GFX11.
1316//===----------------------------------------------------------------------===//
1317
1318let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1319  multiclass DS_Real_gfx7<bits<8> op> {
1320    def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1321                                             SIEncodingFamily.SI>;
1322  }
1323} // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1324
1325multiclass DS_Real_gfx7_gfx10_gfx11<bits<8> op> :
1326  DS_Real_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1327
1328multiclass DS_Real_gfx7_gfx10<bits<8> op> :
1329  DS_Real_gfx7<op>, DS_Real_gfx10<op>;
1330
1331// FIXME-GFX7: Add tests when upstreaming this part.
1332defm DS_GWS_SEMA_RELEASE_ALL : DS_Real_gfx7_gfx10_gfx11<0x018>;
1333defm DS_WRAP_RTN_B32         : DS_Real_gfx7_gfx10_gfx11<0x034>;
1334defm DS_CONDXCHG32_RTN_B64   : DS_Real_gfx7_gfx10_gfx11<0x07e>;
1335defm DS_WRITE_B96            : DS_Real_gfx7_gfx10<0x0de>;
1336defm DS_WRITE_B128           : DS_Real_gfx7_gfx10<0x0df>;
1337defm DS_READ_B96             : DS_Real_gfx7_gfx10<0x0fe>;
1338defm DS_READ_B128            : DS_Real_gfx7_gfx10<0x0ff>;
1339
1340//===----------------------------------------------------------------------===//
1341// GFX6, GFX7, GFX10, GFX11.
1342//===----------------------------------------------------------------------===//
1343
1344let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1345  multiclass DS_Real_gfx6_gfx7<bits<8> op> {
1346    def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
1347                                                  SIEncodingFamily.SI>;
1348  }
1349} // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1350
1351multiclass DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op> :
1352  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10_gfx11<op>;
1353
1354multiclass DS_Real_gfx6_gfx7_gfx10<bits<8> op> :
1355  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10<op>;
1356
1357defm DS_ADD_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x000>;
1358defm DS_SUB_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x001>;
1359defm DS_RSUB_U32            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x002>;
1360defm DS_INC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x003>;
1361defm DS_DEC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x004>;
1362defm DS_MIN_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x005>;
1363defm DS_MAX_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x006>;
1364defm DS_MIN_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x007>;
1365defm DS_MAX_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x008>;
1366defm DS_AND_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x009>;
1367defm DS_OR_B32              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00a>;
1368defm DS_XOR_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00b>;
1369defm DS_MSKOR_B32           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00c>;
1370
1371defm DS_WRITE_B32           : DS_Real_gfx6_gfx7_gfx10<0x00d>;
1372defm DS_WRITE2_B32          : DS_Real_gfx6_gfx7_gfx10<0x00e>;
1373defm DS_WRITE2ST64_B32      : DS_Real_gfx6_gfx7_gfx10<0x00f>;
1374defm DS_CMPST_B32           : DS_Real_gfx6_gfx7_gfx10<0x010>;
1375defm DS_CMPST_F32           : DS_Real_gfx6_gfx7_gfx10<0x011>;
1376
1377defm DS_MIN_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x012>;
1378defm DS_MAX_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x013>;
1379defm DS_NOP                 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x014>;
1380defm DS_GWS_INIT            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x019>;
1381defm DS_GWS_SEMA_V          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01a>;
1382defm DS_GWS_SEMA_BR         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01b>;
1383defm DS_GWS_SEMA_P          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01c>;
1384defm DS_GWS_BARRIER         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01d>;
1385
1386defm DS_WRITE_B8            : DS_Real_gfx6_gfx7_gfx10<0x01e>;
1387defm DS_WRITE_B16           : DS_Real_gfx6_gfx7_gfx10<0x01f>;
1388
1389defm DS_ADD_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x020>;
1390defm DS_SUB_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x021>;
1391defm DS_RSUB_RTN_U32        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x022>;
1392defm DS_INC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x023>;
1393defm DS_DEC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x024>;
1394defm DS_MIN_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x025>;
1395defm DS_MAX_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x026>;
1396defm DS_MIN_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x027>;
1397defm DS_MAX_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x028>;
1398defm DS_AND_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x029>;
1399defm DS_OR_RTN_B32          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02a>;
1400defm DS_XOR_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02b>;
1401defm DS_MSKOR_RTN_B32       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02c>;
1402
1403defm DS_WRXCHG_RTN_B32      : DS_Real_gfx6_gfx7_gfx10<0x02d>;
1404defm DS_WRXCHG2_RTN_B32     : DS_Real_gfx6_gfx7_gfx10<0x02e>;
1405defm DS_WRXCHG2ST64_RTN_B32 : DS_Real_gfx6_gfx7_gfx10<0x02f>;
1406defm DS_CMPST_RTN_B32       : DS_Real_gfx6_gfx7_gfx10<0x030>;
1407defm DS_CMPST_RTN_F32       : DS_Real_gfx6_gfx7_gfx10<0x031>;
1408
1409defm DS_MIN_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x032>;
1410defm DS_MAX_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x033>;
1411defm DS_SWIZZLE_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x035>;
1412
1413defm DS_READ_B32            : DS_Real_gfx6_gfx7_gfx10<0x036>;
1414defm DS_READ2_B32           : DS_Real_gfx6_gfx7_gfx10<0x037>;
1415defm DS_READ2ST64_B32       : DS_Real_gfx6_gfx7_gfx10<0x038>;
1416defm DS_READ_I8             : DS_Real_gfx6_gfx7_gfx10<0x039>;
1417defm DS_READ_U8             : DS_Real_gfx6_gfx7_gfx10<0x03a>;
1418defm DS_READ_I16            : DS_Real_gfx6_gfx7_gfx10<0x03b>;
1419defm DS_READ_U16            : DS_Real_gfx6_gfx7_gfx10<0x03c>;
1420
1421defm DS_CONSUME             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03d>;
1422defm DS_APPEND              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03e>;
1423defm DS_ORDERED_COUNT       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03f>;
1424defm DS_ADD_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x040>;
1425defm DS_SUB_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x041>;
1426defm DS_RSUB_U64            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x042>;
1427defm DS_INC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x043>;
1428defm DS_DEC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x044>;
1429defm DS_MIN_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x045>;
1430defm DS_MAX_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x046>;
1431defm DS_MIN_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x047>;
1432defm DS_MAX_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x048>;
1433defm DS_AND_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x049>;
1434defm DS_OR_B64              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04a>;
1435defm DS_XOR_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04b>;
1436defm DS_MSKOR_B64           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04c>;
1437
1438defm DS_WRITE_B64           : DS_Real_gfx6_gfx7_gfx10<0x04d>;
1439defm DS_WRITE2_B64          : DS_Real_gfx6_gfx7_gfx10<0x04e>;
1440defm DS_WRITE2ST64_B64      : DS_Real_gfx6_gfx7_gfx10<0x04f>;
1441defm DS_CMPST_B64           : DS_Real_gfx6_gfx7_gfx10<0x050>;
1442defm DS_CMPST_F64           : DS_Real_gfx6_gfx7_gfx10<0x051>;
1443
1444defm DS_MIN_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x052>;
1445defm DS_MAX_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x053>;
1446defm DS_ADD_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x060>;
1447defm DS_SUB_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x061>;
1448defm DS_RSUB_RTN_U64        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x062>;
1449defm DS_INC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x063>;
1450defm DS_DEC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x064>;
1451defm DS_MIN_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x065>;
1452defm DS_MAX_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x066>;
1453defm DS_MIN_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x067>;
1454defm DS_MAX_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x068>;
1455defm DS_AND_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x069>;
1456defm DS_OR_RTN_B64          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06a>;
1457defm DS_XOR_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06b>;
1458defm DS_MSKOR_RTN_B64       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06c>;
1459
1460defm DS_WRXCHG_RTN_B64      : DS_Real_gfx6_gfx7_gfx10<0x06d>;
1461defm DS_WRXCHG2_RTN_B64     : DS_Real_gfx6_gfx7_gfx10<0x06e>;
1462defm DS_WRXCHG2ST64_RTN_B64 : DS_Real_gfx6_gfx7_gfx10<0x06f>;
1463defm DS_CMPST_RTN_B64       : DS_Real_gfx6_gfx7_gfx10<0x070>;
1464defm DS_CMPST_RTN_F64       : DS_Real_gfx6_gfx7_gfx10<0x071>;
1465
1466defm DS_MIN_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x072>;
1467defm DS_MAX_RTN_F64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x073>;
1468
1469defm DS_READ_B64            : DS_Real_gfx6_gfx7_gfx10<0x076>;
1470defm DS_READ2_B64           : DS_Real_gfx6_gfx7_gfx10<0x077>;
1471defm DS_READ2ST64_B64       : DS_Real_gfx6_gfx7_gfx10<0x078>;
1472defm DS_ADD_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x080>;
1473defm DS_SUB_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x081>;
1474defm DS_RSUB_SRC2_U32       : DS_Real_gfx6_gfx7_gfx10<0x082>;
1475defm DS_INC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x083>;
1476defm DS_DEC_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x084>;
1477defm DS_MIN_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x085>;
1478defm DS_MAX_SRC2_I32        : DS_Real_gfx6_gfx7_gfx10<0x086>;
1479defm DS_MIN_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x087>;
1480defm DS_MAX_SRC2_U32        : DS_Real_gfx6_gfx7_gfx10<0x088>;
1481defm DS_AND_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x089>;
1482defm DS_OR_SRC2_B32         : DS_Real_gfx6_gfx7_gfx10<0x08a>;
1483defm DS_XOR_SRC2_B32        : DS_Real_gfx6_gfx7_gfx10<0x08b>;
1484defm DS_WRITE_SRC2_B32      : DS_Real_gfx6_gfx7_gfx10<0x08d>;
1485defm DS_MIN_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x092>;
1486defm DS_MAX_SRC2_F32        : DS_Real_gfx6_gfx7_gfx10<0x093>;
1487defm DS_ADD_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c0>;
1488defm DS_SUB_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c1>;
1489defm DS_RSUB_SRC2_U64       : DS_Real_gfx6_gfx7_gfx10<0x0c2>;
1490defm DS_INC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c3>;
1491defm DS_DEC_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c4>;
1492defm DS_MIN_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c5>;
1493defm DS_MAX_SRC2_I64        : DS_Real_gfx6_gfx7_gfx10<0x0c6>;
1494defm DS_MIN_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c7>;
1495defm DS_MAX_SRC2_U64        : DS_Real_gfx6_gfx7_gfx10<0x0c8>;
1496defm DS_AND_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0c9>;
1497defm DS_OR_SRC2_B64         : DS_Real_gfx6_gfx7_gfx10<0x0ca>;
1498defm DS_XOR_SRC2_B64        : DS_Real_gfx6_gfx7_gfx10<0x0cb>;
1499defm DS_WRITE_SRC2_B64      : DS_Real_gfx6_gfx7_gfx10<0x0cd>;
1500defm DS_MIN_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d2>;
1501defm DS_MAX_SRC2_F64        : DS_Real_gfx6_gfx7_gfx10<0x0d3>;
1502
1503//===----------------------------------------------------------------------===//
1504// GFX8, GFX9 (VI).
1505//===----------------------------------------------------------------------===//
1506
1507class DS_Real_vi <bits<8> op, DS_Pseudo ps> :
1508  DS_Real <ps>,
1509  SIMCInstr <ps.Mnemonic, SIEncodingFamily.VI> {
1510  let AssemblerPredicate = isGFX8GFX9;
1511  let DecoderNamespace = "GFX8";
1512
1513  // encoding
1514  let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
1515  let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
1516  let Inst{16}    = !if(ps.has_gds, gds, ps.gdsValue);
1517  let Inst{24-17} = op;
1518  let Inst{25}    = acc;
1519  let Inst{31-26} = 0x36; // ds prefix
1520  let Inst{39-32} = !if(ps.has_addr, addr, !if(ps.has_gws_data0, data0{7-0}, 0));
1521  let Inst{47-40} = !if(ps.has_data0, data0{7-0}, 0);
1522  let Inst{55-48} = !if(ps.has_data1, data1{7-0}, 0);
1523  let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
1524}
1525
1526def DS_ADD_U32_vi         : DS_Real_vi<0x0,  DS_ADD_U32>;
1527def DS_SUB_U32_vi         : DS_Real_vi<0x1,  DS_SUB_U32>;
1528def DS_RSUB_U32_vi        : DS_Real_vi<0x2,  DS_RSUB_U32>;
1529def DS_INC_U32_vi         : DS_Real_vi<0x3,  DS_INC_U32>;
1530def DS_DEC_U32_vi         : DS_Real_vi<0x4,  DS_DEC_U32>;
1531def DS_MIN_I32_vi         : DS_Real_vi<0x5,  DS_MIN_I32>;
1532def DS_MAX_I32_vi         : DS_Real_vi<0x6,  DS_MAX_I32>;
1533def DS_MIN_U32_vi         : DS_Real_vi<0x7,  DS_MIN_U32>;
1534def DS_MAX_U32_vi         : DS_Real_vi<0x8,  DS_MAX_U32>;
1535def DS_AND_B32_vi         : DS_Real_vi<0x9,  DS_AND_B32>;
1536def DS_OR_B32_vi          : DS_Real_vi<0xa,  DS_OR_B32>;
1537def DS_XOR_B32_vi         : DS_Real_vi<0xb,  DS_XOR_B32>;
1538def DS_MSKOR_B32_vi       : DS_Real_vi<0xc,  DS_MSKOR_B32>;
1539def DS_WRITE_B32_vi       : DS_Real_vi<0xd,  DS_WRITE_B32>;
1540def DS_WRITE2_B32_vi      : DS_Real_vi<0xe,  DS_WRITE2_B32>;
1541def DS_WRITE2ST64_B32_vi  : DS_Real_vi<0xf,  DS_WRITE2ST64_B32>;
1542def DS_CMPST_B32_vi       : DS_Real_vi<0x10, DS_CMPST_B32>;
1543def DS_CMPST_F32_vi       : DS_Real_vi<0x11, DS_CMPST_F32>;
1544def DS_MIN_F32_vi         : DS_Real_vi<0x12, DS_MIN_F32>;
1545def DS_MAX_F32_vi         : DS_Real_vi<0x13, DS_MAX_F32>;
1546def DS_NOP_vi             : DS_Real_vi<0x14, DS_NOP>;
1547def DS_ADD_F32_vi         : DS_Real_vi<0x15, DS_ADD_F32>;
1548def DS_GWS_INIT_vi        : DS_Real_vi<0x99, DS_GWS_INIT>;
1549def DS_GWS_SEMA_V_vi      : DS_Real_vi<0x9a, DS_GWS_SEMA_V>;
1550def DS_GWS_SEMA_BR_vi     : DS_Real_vi<0x9b, DS_GWS_SEMA_BR>;
1551def DS_GWS_SEMA_P_vi      : DS_Real_vi<0x9c, DS_GWS_SEMA_P>;
1552def DS_GWS_BARRIER_vi     : DS_Real_vi<0x9d, DS_GWS_BARRIER>;
1553def DS_WRITE_ADDTID_B32_vi : DS_Real_vi<0x1d, DS_WRITE_ADDTID_B32>;
1554def DS_WRITE_B8_vi        : DS_Real_vi<0x1e, DS_WRITE_B8>;
1555def DS_WRITE_B16_vi       : DS_Real_vi<0x1f, DS_WRITE_B16>;
1556def DS_ADD_RTN_U32_vi     : DS_Real_vi<0x20, DS_ADD_RTN_U32>;
1557def DS_SUB_RTN_U32_vi     : DS_Real_vi<0x21, DS_SUB_RTN_U32>;
1558def DS_RSUB_RTN_U32_vi    : DS_Real_vi<0x22, DS_RSUB_RTN_U32>;
1559def DS_INC_RTN_U32_vi     : DS_Real_vi<0x23, DS_INC_RTN_U32>;
1560def DS_DEC_RTN_U32_vi     : DS_Real_vi<0x24, DS_DEC_RTN_U32>;
1561def DS_MIN_RTN_I32_vi     : DS_Real_vi<0x25, DS_MIN_RTN_I32>;
1562def DS_MAX_RTN_I32_vi     : DS_Real_vi<0x26, DS_MAX_RTN_I32>;
1563def DS_MIN_RTN_U32_vi     : DS_Real_vi<0x27, DS_MIN_RTN_U32>;
1564def DS_MAX_RTN_U32_vi     : DS_Real_vi<0x28, DS_MAX_RTN_U32>;
1565def DS_AND_RTN_B32_vi     : DS_Real_vi<0x29, DS_AND_RTN_B32>;
1566def DS_OR_RTN_B32_vi      : DS_Real_vi<0x2a, DS_OR_RTN_B32>;
1567def DS_XOR_RTN_B32_vi     : DS_Real_vi<0x2b, DS_XOR_RTN_B32>;
1568def DS_MSKOR_RTN_B32_vi   : DS_Real_vi<0x2c, DS_MSKOR_RTN_B32>;
1569def DS_WRXCHG_RTN_B32_vi  : DS_Real_vi<0x2d, DS_WRXCHG_RTN_B32>;
1570def DS_WRXCHG2_RTN_B32_vi : DS_Real_vi<0x2e, DS_WRXCHG2_RTN_B32>;
1571def DS_WRXCHG2ST64_RTN_B32_vi : DS_Real_vi<0x2f, DS_WRXCHG2ST64_RTN_B32>;
1572def DS_CMPST_RTN_B32_vi   : DS_Real_vi<0x30, DS_CMPST_RTN_B32>;
1573def DS_CMPST_RTN_F32_vi   : DS_Real_vi<0x31, DS_CMPST_RTN_F32>;
1574def DS_MIN_RTN_F32_vi     : DS_Real_vi<0x32, DS_MIN_RTN_F32>;
1575def DS_MAX_RTN_F32_vi     : DS_Real_vi<0x33, DS_MAX_RTN_F32>;
1576def DS_WRAP_RTN_B32_vi    : DS_Real_vi<0x34, DS_WRAP_RTN_B32>;
1577def DS_ADD_RTN_F32_vi     : DS_Real_vi<0x35, DS_ADD_RTN_F32>;
1578def DS_READ_B32_vi        : DS_Real_vi<0x36, DS_READ_B32>;
1579def DS_READ2_B32_vi       : DS_Real_vi<0x37, DS_READ2_B32>;
1580def DS_READ2ST64_B32_vi   : DS_Real_vi<0x38, DS_READ2ST64_B32>;
1581def DS_READ_I8_vi         : DS_Real_vi<0x39, DS_READ_I8>;
1582def DS_READ_U8_vi         : DS_Real_vi<0x3a, DS_READ_U8>;
1583def DS_READ_I16_vi        : DS_Real_vi<0x3b, DS_READ_I16>;
1584def DS_READ_U16_vi        : DS_Real_vi<0x3c, DS_READ_U16>;
1585def DS_READ_ADDTID_B32_vi : DS_Real_vi<0xb6, DS_READ_ADDTID_B32>;
1586def DS_CONSUME_vi         : DS_Real_vi<0xbd, DS_CONSUME>;
1587def DS_APPEND_vi          : DS_Real_vi<0xbe, DS_APPEND>;
1588def DS_ORDERED_COUNT_vi   : DS_Real_vi<0xbf, DS_ORDERED_COUNT>;
1589def DS_SWIZZLE_B32_vi     : DS_Real_vi<0x3d, DS_SWIZZLE_B32>;
1590def DS_PERMUTE_B32_vi     : DS_Real_vi<0x3e, DS_PERMUTE_B32>;
1591def DS_BPERMUTE_B32_vi    : DS_Real_vi<0x3f, DS_BPERMUTE_B32>;
1592
1593def DS_ADD_U64_vi         : DS_Real_vi<0x40, DS_ADD_U64>;
1594def DS_SUB_U64_vi         : DS_Real_vi<0x41, DS_SUB_U64>;
1595def DS_RSUB_U64_vi        : DS_Real_vi<0x42, DS_RSUB_U64>;
1596def DS_INC_U64_vi         : DS_Real_vi<0x43, DS_INC_U64>;
1597def DS_DEC_U64_vi         : DS_Real_vi<0x44, DS_DEC_U64>;
1598def DS_MIN_I64_vi         : DS_Real_vi<0x45, DS_MIN_I64>;
1599def DS_MAX_I64_vi         : DS_Real_vi<0x46, DS_MAX_I64>;
1600def DS_MIN_U64_vi         : DS_Real_vi<0x47, DS_MIN_U64>;
1601def DS_MAX_U64_vi         : DS_Real_vi<0x48, DS_MAX_U64>;
1602def DS_AND_B64_vi         : DS_Real_vi<0x49, DS_AND_B64>;
1603def DS_OR_B64_vi          : DS_Real_vi<0x4a, DS_OR_B64>;
1604def DS_XOR_B64_vi         : DS_Real_vi<0x4b, DS_XOR_B64>;
1605def DS_MSKOR_B64_vi       : DS_Real_vi<0x4c, DS_MSKOR_B64>;
1606def DS_WRITE_B64_vi       : DS_Real_vi<0x4d, DS_WRITE_B64>;
1607def DS_WRITE2_B64_vi      : DS_Real_vi<0x4E, DS_WRITE2_B64>;
1608def DS_WRITE2ST64_B64_vi  : DS_Real_vi<0x4f, DS_WRITE2ST64_B64>;
1609def DS_CMPST_B64_vi       : DS_Real_vi<0x50, DS_CMPST_B64>;
1610def DS_CMPST_F64_vi       : DS_Real_vi<0x51, DS_CMPST_F64>;
1611def DS_MIN_F64_vi         : DS_Real_vi<0x52, DS_MIN_F64>;
1612def DS_MAX_F64_vi         : DS_Real_vi<0x53, DS_MAX_F64>;
1613
1614def DS_WRITE_B8_D16_HI_vi  : DS_Real_vi<0x54, DS_WRITE_B8_D16_HI>;
1615def DS_WRITE_B16_D16_HI_vi : DS_Real_vi<0x55, DS_WRITE_B16_D16_HI>;
1616
1617def DS_READ_U8_D16_vi     : DS_Real_vi<0x56, DS_READ_U8_D16>;
1618def DS_READ_U8_D16_HI_vi  : DS_Real_vi<0x57, DS_READ_U8_D16_HI>;
1619def DS_READ_I8_D16_vi     : DS_Real_vi<0x58, DS_READ_I8_D16>;
1620def DS_READ_I8_D16_HI_vi  : DS_Real_vi<0x59, DS_READ_I8_D16_HI>;
1621def DS_READ_U16_D16_vi    : DS_Real_vi<0x5a, DS_READ_U16_D16>;
1622def DS_READ_U16_D16_HI_vi : DS_Real_vi<0x5b, DS_READ_U16_D16_HI>;
1623
1624def DS_ADD_RTN_U64_vi     : DS_Real_vi<0x60, DS_ADD_RTN_U64>;
1625def DS_SUB_RTN_U64_vi     : DS_Real_vi<0x61, DS_SUB_RTN_U64>;
1626def DS_RSUB_RTN_U64_vi    : DS_Real_vi<0x62, DS_RSUB_RTN_U64>;
1627def DS_INC_RTN_U64_vi     : DS_Real_vi<0x63, DS_INC_RTN_U64>;
1628def DS_DEC_RTN_U64_vi     : DS_Real_vi<0x64, DS_DEC_RTN_U64>;
1629def DS_MIN_RTN_I64_vi     : DS_Real_vi<0x65, DS_MIN_RTN_I64>;
1630def DS_MAX_RTN_I64_vi     : DS_Real_vi<0x66, DS_MAX_RTN_I64>;
1631def DS_MIN_RTN_U64_vi     : DS_Real_vi<0x67, DS_MIN_RTN_U64>;
1632def DS_MAX_RTN_U64_vi     : DS_Real_vi<0x68, DS_MAX_RTN_U64>;
1633def DS_AND_RTN_B64_vi     : DS_Real_vi<0x69, DS_AND_RTN_B64>;
1634def DS_OR_RTN_B64_vi      : DS_Real_vi<0x6a, DS_OR_RTN_B64>;
1635def DS_XOR_RTN_B64_vi     : DS_Real_vi<0x6b, DS_XOR_RTN_B64>;
1636def DS_MSKOR_RTN_B64_vi   : DS_Real_vi<0x6c, DS_MSKOR_RTN_B64>;
1637def DS_WRXCHG_RTN_B64_vi  : DS_Real_vi<0x6d, DS_WRXCHG_RTN_B64>;
1638def DS_WRXCHG2_RTN_B64_vi : DS_Real_vi<0x6e, DS_WRXCHG2_RTN_B64>;
1639def DS_WRXCHG2ST64_RTN_B64_vi : DS_Real_vi<0x6f, DS_WRXCHG2ST64_RTN_B64>;
1640def DS_CONDXCHG32_RTN_B64_vi   : DS_Real_vi<0x7e, DS_CONDXCHG32_RTN_B64>;
1641def DS_GWS_SEMA_RELEASE_ALL_vi : DS_Real_vi<0x98, DS_GWS_SEMA_RELEASE_ALL>;
1642def DS_CMPST_RTN_B64_vi   : DS_Real_vi<0x70, DS_CMPST_RTN_B64>;
1643def DS_CMPST_RTN_F64_vi   : DS_Real_vi<0x71, DS_CMPST_RTN_F64>;
1644def DS_MIN_RTN_F64_vi     : DS_Real_vi<0x72, DS_MIN_RTN_F64>;
1645def DS_MAX_RTN_F64_vi     : DS_Real_vi<0x73, DS_MAX_RTN_F64>;
1646
1647def DS_READ_B64_vi        : DS_Real_vi<0x76, DS_READ_B64>;
1648def DS_READ2_B64_vi       : DS_Real_vi<0x77, DS_READ2_B64>;
1649def DS_READ2ST64_B64_vi   : DS_Real_vi<0x78, DS_READ2ST64_B64>;
1650
1651def DS_ADD_SRC2_U32_vi    : DS_Real_vi<0x80, DS_ADD_SRC2_U32>;
1652def DS_SUB_SRC2_U32_vi    : DS_Real_vi<0x81, DS_SUB_SRC2_U32>;
1653def DS_RSUB_SRC2_U32_vi   : DS_Real_vi<0x82, DS_RSUB_SRC2_U32>;
1654def DS_INC_SRC2_U32_vi    : DS_Real_vi<0x83, DS_INC_SRC2_U32>;
1655def DS_DEC_SRC2_U32_vi    : DS_Real_vi<0x84, DS_DEC_SRC2_U32>;
1656def DS_MIN_SRC2_I32_vi    : DS_Real_vi<0x85, DS_MIN_SRC2_I32>;
1657def DS_MAX_SRC2_I32_vi    : DS_Real_vi<0x86, DS_MAX_SRC2_I32>;
1658def DS_MIN_SRC2_U32_vi    : DS_Real_vi<0x87, DS_MIN_SRC2_U32>;
1659def DS_MAX_SRC2_U32_vi    : DS_Real_vi<0x88, DS_MAX_SRC2_U32>;
1660def DS_AND_SRC2_B32_vi    : DS_Real_vi<0x89, DS_AND_SRC2_B32>;
1661def DS_OR_SRC2_B32_vi     : DS_Real_vi<0x8a, DS_OR_SRC2_B32>;
1662def DS_XOR_SRC2_B32_vi    : DS_Real_vi<0x8b, DS_XOR_SRC2_B32>;
1663def DS_WRITE_SRC2_B32_vi  : DS_Real_vi<0x8d, DS_WRITE_SRC2_B32>;
1664def DS_MIN_SRC2_F32_vi    : DS_Real_vi<0x92, DS_MIN_SRC2_F32>;
1665def DS_MAX_SRC2_F32_vi    : DS_Real_vi<0x93, DS_MAX_SRC2_F32>;
1666def DS_ADD_SRC2_F32_vi    : DS_Real_vi<0x95, DS_ADD_SRC2_F32>;
1667def DS_ADD_SRC2_U64_vi    : DS_Real_vi<0xc0, DS_ADD_SRC2_U64>;
1668def DS_SUB_SRC2_U64_vi    : DS_Real_vi<0xc1, DS_SUB_SRC2_U64>;
1669def DS_RSUB_SRC2_U64_vi   : DS_Real_vi<0xc2, DS_RSUB_SRC2_U64>;
1670def DS_INC_SRC2_U64_vi    : DS_Real_vi<0xc3, DS_INC_SRC2_U64>;
1671def DS_DEC_SRC2_U64_vi    : DS_Real_vi<0xc4, DS_DEC_SRC2_U64>;
1672def DS_MIN_SRC2_I64_vi    : DS_Real_vi<0xc5, DS_MIN_SRC2_I64>;
1673def DS_MAX_SRC2_I64_vi    : DS_Real_vi<0xc6, DS_MAX_SRC2_I64>;
1674def DS_MIN_SRC2_U64_vi    : DS_Real_vi<0xc7, DS_MIN_SRC2_U64>;
1675def DS_MAX_SRC2_U64_vi    : DS_Real_vi<0xc8, DS_MAX_SRC2_U64>;
1676def DS_AND_SRC2_B64_vi    : DS_Real_vi<0xc9, DS_AND_SRC2_B64>;
1677def DS_OR_SRC2_B64_vi     : DS_Real_vi<0xca, DS_OR_SRC2_B64>;
1678def DS_XOR_SRC2_B64_vi    : DS_Real_vi<0xcb, DS_XOR_SRC2_B64>;
1679def DS_WRITE_SRC2_B64_vi  : DS_Real_vi<0xcd, DS_WRITE_SRC2_B64>;
1680def DS_MIN_SRC2_F64_vi    : DS_Real_vi<0xd2, DS_MIN_SRC2_F64>;
1681def DS_MAX_SRC2_F64_vi    : DS_Real_vi<0xd3, DS_MAX_SRC2_F64>;
1682def DS_WRITE_B96_vi       : DS_Real_vi<0xde, DS_WRITE_B96>;
1683def DS_WRITE_B128_vi      : DS_Real_vi<0xdf, DS_WRITE_B128>;
1684def DS_READ_B96_vi        : DS_Real_vi<0xfe, DS_READ_B96>;
1685def DS_READ_B128_vi       : DS_Real_vi<0xff, DS_READ_B128>;
1686
1687let SubtargetPredicate = isGFX90APlus in {
1688  def DS_ADD_F64_vi     : DS_Real_vi<0x5c, DS_ADD_F64>;
1689  def DS_ADD_RTN_F64_vi : DS_Real_vi<0x7c, DS_ADD_RTN_F64>;
1690} // End SubtargetPredicate = isGFX90APlus
1691
1692let SubtargetPredicate = isGFX940Plus in {
1693  def DS_PK_ADD_F16_vi     : DS_Real_vi<0x17, DS_PK_ADD_F16>;
1694  def DS_PK_ADD_RTN_F16_vi : DS_Real_vi<0xb7, DS_PK_ADD_RTN_F16>;
1695  def DS_PK_ADD_BF16_vi     : DS_Real_vi<0x18, DS_PK_ADD_BF16>;
1696  def DS_PK_ADD_RTN_BF16_vi : DS_Real_vi<0xb8, DS_PK_ADD_RTN_BF16>;
1697} // End SubtargetPredicate = isGFX940Plus
1698