xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp (revision 700637cbb5e582861067a11aaca4d053546871d2) 
1 //===-- R600InstrInfo.cpp - R600 Instruction Information ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// R600 Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "R600InstrInfo.h"
15 #include "MCTargetDesc/R600MCTargetDesc.h"
16 #include "R600Defines.h"
17 #include "R600Subtarget.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/CodeGen/MachineFrameInfo.h"
20 
21 using namespace llvm;
22 
23 #define GET_INSTRINFO_CTOR_DTOR
24 #include "R600GenDFAPacketizer.inc"
25 
26 #define GET_INSTRINFO_CTOR_DTOR
27 #define GET_INSTRMAP_INFO
28 #define GET_INSTRINFO_NAMED_OPS
29 #include "R600GenInstrInfo.inc"
30 
R600InstrInfo(const R600Subtarget & ST)31 R600InstrInfo::R600InstrInfo(const R600Subtarget &ST)
32   : R600GenInstrInfo(-1, -1), RI(), ST(ST) {}
33 
isVector(const MachineInstr & MI) const34 bool R600InstrInfo::isVector(const MachineInstr &MI) const {
35   return get(MI.getOpcode()).TSFlags & R600_InstFlag::VECTOR;
36 }
37 
copyPhysReg(MachineBasicBlock & MBB,MachineBasicBlock::iterator MI,const DebugLoc & DL,Register DestReg,Register SrcReg,bool KillSrc,bool RenamableDest,bool RenamableSrc) const38 void R600InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
39                                 MachineBasicBlock::iterator MI,
40                                 const DebugLoc &DL, Register DestReg,
41                                 Register SrcReg, bool KillSrc,
42                                 bool RenamableDest, bool RenamableSrc) const {
43   unsigned VectorComponents = 0;
44   if ((R600::R600_Reg128RegClass.contains(DestReg) ||
45       R600::R600_Reg128VerticalRegClass.contains(DestReg)) &&
46       (R600::R600_Reg128RegClass.contains(SrcReg) ||
47        R600::R600_Reg128VerticalRegClass.contains(SrcReg))) {
48     VectorComponents = 4;
49   } else if((R600::R600_Reg64RegClass.contains(DestReg) ||
50             R600::R600_Reg64VerticalRegClass.contains(DestReg)) &&
51             (R600::R600_Reg64RegClass.contains(SrcReg) ||
52              R600::R600_Reg64VerticalRegClass.contains(SrcReg))) {
53     VectorComponents = 2;
54   }
55 
56   if (VectorComponents > 0) {
57     for (unsigned I = 0; I < VectorComponents; I++) {
58       unsigned SubRegIndex = R600RegisterInfo::getSubRegFromChannel(I);
59       buildDefaultInstruction(MBB, MI, R600::MOV,
60                               RI.getSubReg(DestReg, SubRegIndex),
61                               RI.getSubReg(SrcReg, SubRegIndex))
62                               .addReg(DestReg,
63                                       RegState::Define | RegState::Implicit);
64     }
65   } else {
66     MachineInstr *NewMI = buildDefaultInstruction(MBB, MI, R600::MOV,
67                                                   DestReg, SrcReg);
68     NewMI->getOperand(getOperandIdx(*NewMI, R600::OpName::src0))
69                                     .setIsKill(KillSrc);
70   }
71 }
72 
73 /// \returns true if \p MBBI can be moved into a new basic.
isLegalToSplitMBBAt(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const74 bool R600InstrInfo::isLegalToSplitMBBAt(MachineBasicBlock &MBB,
75                                        MachineBasicBlock::iterator MBBI) const {
76   for (const MachineOperand &MO : MBBI->all_uses())
77     if (!MO.getReg().isVirtual() && RI.isPhysRegLiveAcrossClauses(MO.getReg()))
78       return false;
79   return true;
80 }
81 
isMov(unsigned Opcode) const82 bool R600InstrInfo::isMov(unsigned Opcode) const {
83   switch(Opcode) {
84   default:
85     return false;
86   case R600::MOV:
87   case R600::MOV_IMM_F32:
88   case R600::MOV_IMM_I32:
89     return true;
90   }
91 }
92 
isReductionOp(unsigned Opcode) const93 bool R600InstrInfo::isReductionOp(unsigned Opcode) const {
94   return false;
95 }
96 
isCubeOp(unsigned Opcode) const97 bool R600InstrInfo::isCubeOp(unsigned Opcode) const {
98   switch(Opcode) {
99     default: return false;
100     case R600::CUBE_r600_pseudo:
101     case R600::CUBE_r600_real:
102     case R600::CUBE_eg_pseudo:
103     case R600::CUBE_eg_real:
104       return true;
105   }
106 }
107 
isALUInstr(unsigned Opcode) const108 bool R600InstrInfo::isALUInstr(unsigned Opcode) const {
109   unsigned TargetFlags = get(Opcode).TSFlags;
110 
111   return (TargetFlags & R600_InstFlag::ALU_INST);
112 }
113 
hasInstrModifiers(unsigned Opcode) const114 bool R600InstrInfo::hasInstrModifiers(unsigned Opcode) const {
115   unsigned TargetFlags = get(Opcode).TSFlags;
116 
117   return ((TargetFlags & R600_InstFlag::OP1) |
118           (TargetFlags & R600_InstFlag::OP2) |
119           (TargetFlags & R600_InstFlag::OP3));
120 }
121 
isLDSInstr(unsigned Opcode) const122 bool R600InstrInfo::isLDSInstr(unsigned Opcode) const {
123   unsigned TargetFlags = get(Opcode).TSFlags;
124 
125   return ((TargetFlags & R600_InstFlag::LDS_1A) |
126           (TargetFlags & R600_InstFlag::LDS_1A1D) |
127           (TargetFlags & R600_InstFlag::LDS_1A2D));
128 }
129 
isLDSRetInstr(unsigned Opcode) const130 bool R600InstrInfo::isLDSRetInstr(unsigned Opcode) const {
131   return isLDSInstr(Opcode) && getOperandIdx(Opcode, R600::OpName::dst) != -1;
132 }
133 
canBeConsideredALU(const MachineInstr & MI) const134 bool R600InstrInfo::canBeConsideredALU(const MachineInstr &MI) const {
135   if (isALUInstr(MI.getOpcode()))
136     return true;
137   if (isVector(MI) || isCubeOp(MI.getOpcode()))
138     return true;
139   switch (MI.getOpcode()) {
140   case R600::PRED_X:
141   case R600::INTERP_PAIR_XY:
142   case R600::INTERP_PAIR_ZW:
143   case R600::INTERP_VEC_LOAD:
144   case R600::COPY:
145   case R600::DOT_4:
146     return true;
147   default:
148     return false;
149   }
150 }
151 
isTransOnly(unsigned Opcode) const152 bool R600InstrInfo::isTransOnly(unsigned Opcode) const {
153   if (ST.hasCaymanISA())
154     return false;
155   return (get(Opcode).getSchedClass() == R600::Sched::TransALU);
156 }
157 
isTransOnly(const MachineInstr & MI) const158 bool R600InstrInfo::isTransOnly(const MachineInstr &MI) const {
159   return isTransOnly(MI.getOpcode());
160 }
161 
isVectorOnly(unsigned Opcode) const162 bool R600InstrInfo::isVectorOnly(unsigned Opcode) const {
163   return (get(Opcode).getSchedClass() == R600::Sched::VecALU);
164 }
165 
isVectorOnly(const MachineInstr & MI) const166 bool R600InstrInfo::isVectorOnly(const MachineInstr &MI) const {
167   return isVectorOnly(MI.getOpcode());
168 }
169 
isExport(unsigned Opcode) const170 bool R600InstrInfo::isExport(unsigned Opcode) const {
171   return (get(Opcode).TSFlags & R600_InstFlag::IS_EXPORT);
172 }
173 
usesVertexCache(unsigned Opcode) const174 bool R600InstrInfo::usesVertexCache(unsigned Opcode) const {
175   return ST.hasVertexCache() && IS_VTX(get(Opcode));
176 }
177 
usesVertexCache(const MachineInstr & MI) const178 bool R600InstrInfo::usesVertexCache(const MachineInstr &MI) const {
179   const MachineFunction *MF = MI.getParent()->getParent();
180   return !AMDGPU::isCompute(MF->getFunction().getCallingConv()) &&
181          usesVertexCache(MI.getOpcode());
182 }
183 
usesTextureCache(unsigned Opcode) const184 bool R600InstrInfo::usesTextureCache(unsigned Opcode) const {
185   return (!ST.hasVertexCache() && IS_VTX(get(Opcode))) || IS_TEX(get(Opcode));
186 }
187 
usesTextureCache(const MachineInstr & MI) const188 bool R600InstrInfo::usesTextureCache(const MachineInstr &MI) const {
189   const MachineFunction *MF = MI.getParent()->getParent();
190   return (AMDGPU::isCompute(MF->getFunction().getCallingConv()) &&
191           usesVertexCache(MI.getOpcode())) ||
192           usesTextureCache(MI.getOpcode());
193 }
194 
mustBeLastInClause(unsigned Opcode) const195 bool R600InstrInfo::mustBeLastInClause(unsigned Opcode) const {
196   switch (Opcode) {
197   case R600::KILLGT:
198   case R600::GROUP_BARRIER:
199     return true;
200   default:
201     return false;
202   }
203 }
204 
usesAddressRegister(MachineInstr & MI) const205 bool R600InstrInfo::usesAddressRegister(MachineInstr &MI) const {
206   return MI.findRegisterUseOperandIdx(R600::AR_X, &RI, false) != -1;
207 }
208 
definesAddressRegister(MachineInstr & MI) const209 bool R600InstrInfo::definesAddressRegister(MachineInstr &MI) const {
210   return MI.findRegisterDefOperandIdx(R600::AR_X, &RI, false, false) != -1;
211 }
212 
readsLDSSrcReg(const MachineInstr & MI) const213 bool R600InstrInfo::readsLDSSrcReg(const MachineInstr &MI) const {
214   if (!isALUInstr(MI.getOpcode())) {
215     return false;
216   }
217   for (const MachineOperand &MO : MI.all_uses())
218     if (MO.getReg().isPhysical() &&
219         R600::R600_LDS_SRC_REGRegClass.contains(MO.getReg()))
220       return true;
221   return false;
222 }
223 
getSelIdx(unsigned Opcode,unsigned SrcIdx) const224 int R600InstrInfo::getSelIdx(unsigned Opcode, unsigned SrcIdx) const {
225   static const R600::OpName SrcSelTable[][2] = {
226       {R600::OpName::src0, R600::OpName::src0_sel},
227       {R600::OpName::src1, R600::OpName::src1_sel},
228       {R600::OpName::src2, R600::OpName::src2_sel},
229       {R600::OpName::src0_X, R600::OpName::src0_sel_X},
230       {R600::OpName::src0_Y, R600::OpName::src0_sel_Y},
231       {R600::OpName::src0_Z, R600::OpName::src0_sel_Z},
232       {R600::OpName::src0_W, R600::OpName::src0_sel_W},
233       {R600::OpName::src1_X, R600::OpName::src1_sel_X},
234       {R600::OpName::src1_Y, R600::OpName::src1_sel_Y},
235       {R600::OpName::src1_Z, R600::OpName::src1_sel_Z},
236       {R600::OpName::src1_W, R600::OpName::src1_sel_W}};
237 
238   for (const auto &Row : SrcSelTable) {
239     if (getOperandIdx(Opcode, Row[0]) == (int)SrcIdx) {
240       return getOperandIdx(Opcode, Row[1]);
241     }
242   }
243   return -1;
244 }
245 
246 SmallVector<std::pair<MachineOperand *, int64_t>, 3>
getSrcs(MachineInstr & MI) const247 R600InstrInfo::getSrcs(MachineInstr &MI) const {
248   SmallVector<std::pair<MachineOperand *, int64_t>, 3> Result;
249 
250   if (MI.getOpcode() == R600::DOT_4) {
251     static const R600::OpName OpTable[8][2] = {
252         {R600::OpName::src0_X, R600::OpName::src0_sel_X},
253         {R600::OpName::src0_Y, R600::OpName::src0_sel_Y},
254         {R600::OpName::src0_Z, R600::OpName::src0_sel_Z},
255         {R600::OpName::src0_W, R600::OpName::src0_sel_W},
256         {R600::OpName::src1_X, R600::OpName::src1_sel_X},
257         {R600::OpName::src1_Y, R600::OpName::src1_sel_Y},
258         {R600::OpName::src1_Z, R600::OpName::src1_sel_Z},
259         {R600::OpName::src1_W, R600::OpName::src1_sel_W},
260     };
261 
262     for (const auto &Op : OpTable) {
263       MachineOperand &MO = MI.getOperand(getOperandIdx(MI.getOpcode(), Op[0]));
264       Register Reg = MO.getReg();
265       if (Reg == R600::ALU_CONST) {
266         MachineOperand &Sel =
267             MI.getOperand(getOperandIdx(MI.getOpcode(), Op[1]));
268         Result.push_back(std::pair(&MO, Sel.getImm()));
269         continue;
270       }
271     }
272     return Result;
273   }
274 
275   static const R600::OpName OpTable[3][2] = {
276       {R600::OpName::src0, R600::OpName::src0_sel},
277       {R600::OpName::src1, R600::OpName::src1_sel},
278       {R600::OpName::src2, R600::OpName::src2_sel},
279   };
280 
281   for (const auto &Op : OpTable) {
282     int SrcIdx = getOperandIdx(MI.getOpcode(), Op[0]);
283     if (SrcIdx < 0)
284       break;
285     MachineOperand &MO = MI.getOperand(SrcIdx);
286     Register Reg = MO.getReg();
287     if (Reg == R600::ALU_CONST) {
288       MachineOperand &Sel = MI.getOperand(getOperandIdx(MI.getOpcode(), Op[1]));
289       Result.push_back(std::pair(&MO, Sel.getImm()));
290       continue;
291     }
292     if (Reg == R600::ALU_LITERAL_X) {
293       MachineOperand &Operand =
294           MI.getOperand(getOperandIdx(MI.getOpcode(), R600::OpName::literal));
295       if (Operand.isImm()) {
296         Result.push_back(std::pair(&MO, Operand.getImm()));
297         continue;
298       }
299       assert(Operand.isGlobal());
300     }
301     Result.push_back(std::pair(&MO, 0));
302   }
303   return Result;
304 }
305 
306 std::vector<std::pair<int, unsigned>>
ExtractSrcs(MachineInstr & MI,const DenseMap<unsigned,unsigned> & PV,unsigned & ConstCount) const307 R600InstrInfo::ExtractSrcs(MachineInstr &MI,
308                            const DenseMap<unsigned, unsigned> &PV,
309                            unsigned &ConstCount) const {
310   ConstCount = 0;
311   const std::pair<int, unsigned> DummyPair(-1, 0);
312   std::vector<std::pair<int, unsigned>> Result;
313   unsigned i = 0;
314   for (const auto &Src : getSrcs(MI)) {
315     ++i;
316     Register Reg = Src.first->getReg();
317     int Index = RI.getEncodingValue(Reg) & 0xff;
318     if (Reg == R600::OQAP) {
319       Result.emplace_back(Index, 0U);
320     }
321     if (PV.contains(Reg)) {
322       // 255 is used to tells its a PS/PV reg
323       Result.emplace_back(255, 0U);
324       continue;
325     }
326     if (Index > 127) {
327       ConstCount++;
328       Result.push_back(DummyPair);
329       continue;
330     }
331     unsigned Chan = RI.getHWRegChan(Reg);
332     Result.emplace_back(Index, Chan);
333   }
334   for (; i < 3; ++i)
335     Result.push_back(DummyPair);
336   return Result;
337 }
338 
339 static std::vector<std::pair<int, unsigned>>
Swizzle(std::vector<std::pair<int,unsigned>> Src,R600InstrInfo::BankSwizzle Swz)340 Swizzle(std::vector<std::pair<int, unsigned>> Src,
341         R600InstrInfo::BankSwizzle Swz) {
342   if (Src[0] == Src[1])
343     Src[1].first = -1;
344   switch (Swz) {
345   case R600InstrInfo::ALU_VEC_012_SCL_210:
346     break;
347   case R600InstrInfo::ALU_VEC_021_SCL_122:
348     std::swap(Src[1], Src[2]);
349     break;
350   case R600InstrInfo::ALU_VEC_102_SCL_221:
351     std::swap(Src[0], Src[1]);
352     break;
353   case R600InstrInfo::ALU_VEC_120_SCL_212:
354     std::swap(Src[0], Src[1]);
355     std::swap(Src[0], Src[2]);
356     break;
357   case R600InstrInfo::ALU_VEC_201:
358     std::swap(Src[0], Src[2]);
359     std::swap(Src[0], Src[1]);
360     break;
361   case R600InstrInfo::ALU_VEC_210:
362     std::swap(Src[0], Src[2]);
363     break;
364   }
365   return Src;
366 }
367 
getTransSwizzle(R600InstrInfo::BankSwizzle Swz,unsigned Op)368 static unsigned getTransSwizzle(R600InstrInfo::BankSwizzle Swz, unsigned Op) {
369   assert(Op < 3 && "Out of range swizzle index");
370   switch (Swz) {
371   case R600InstrInfo::ALU_VEC_012_SCL_210: {
372     unsigned Cycles[3] = { 2, 1, 0};
373     return Cycles[Op];
374   }
375   case R600InstrInfo::ALU_VEC_021_SCL_122: {
376     unsigned Cycles[3] = { 1, 2, 2};
377     return Cycles[Op];
378   }
379   case R600InstrInfo::ALU_VEC_120_SCL_212: {
380     unsigned Cycles[3] = { 2, 1, 2};
381     return Cycles[Op];
382   }
383   case R600InstrInfo::ALU_VEC_102_SCL_221: {
384     unsigned Cycles[3] = { 2, 2, 1};
385     return Cycles[Op];
386   }
387   default:
388     llvm_unreachable("Wrong Swizzle for Trans Slot");
389   }
390 }
391 
392 /// returns how many MIs (whose inputs are represented by IGSrcs) can be packed
393 /// in the same Instruction Group while meeting read port limitations given a
394 /// Swz swizzle sequence.
isLegalUpTo(const std::vector<std::vector<std::pair<int,unsigned>>> & IGSrcs,const std::vector<R600InstrInfo::BankSwizzle> & Swz,const std::vector<std::pair<int,unsigned>> & TransSrcs,R600InstrInfo::BankSwizzle TransSwz) const395 unsigned  R600InstrInfo::isLegalUpTo(
396     const std::vector<std::vector<std::pair<int, unsigned>>> &IGSrcs,
397     const std::vector<R600InstrInfo::BankSwizzle> &Swz,
398     const std::vector<std::pair<int, unsigned>> &TransSrcs,
399     R600InstrInfo::BankSwizzle TransSwz) const {
400   int Vector[4][3];
401   memset(Vector, -1, sizeof(Vector));
402   for (unsigned i = 0, e = IGSrcs.size(); i < e; i++) {
403     const std::vector<std::pair<int, unsigned>> &Srcs =
404         Swizzle(IGSrcs[i], Swz[i]);
405     for (unsigned j = 0; j < 3; j++) {
406       const std::pair<int, unsigned> &Src = Srcs[j];
407       if (Src.first < 0 || Src.first == 255)
408         continue;
409       if (Src.first == GET_REG_INDEX(RI.getEncodingValue(R600::OQAP))) {
410         if (Swz[i] != R600InstrInfo::ALU_VEC_012_SCL_210 &&
411             Swz[i] != R600InstrInfo::ALU_VEC_021_SCL_122) {
412             // The value from output queue A (denoted by register OQAP) can
413             // only be fetched during the first cycle.
414             return false;
415         }
416         // OQAP does not count towards the normal read port restrictions
417         continue;
418       }
419       if (Vector[Src.second][j] < 0)
420         Vector[Src.second][j] = Src.first;
421       if (Vector[Src.second][j] != Src.first)
422         return i;
423     }
424   }
425   // Now check Trans Alu
426   for (unsigned i = 0, e = TransSrcs.size(); i < e; ++i) {
427     const std::pair<int, unsigned> &Src = TransSrcs[i];
428     unsigned Cycle = getTransSwizzle(TransSwz, i);
429     if (Src.first < 0)
430       continue;
431     if (Src.first == 255)
432       continue;
433     if (Vector[Src.second][Cycle] < 0)
434       Vector[Src.second][Cycle] = Src.first;
435     if (Vector[Src.second][Cycle] != Src.first)
436       return IGSrcs.size() - 1;
437   }
438   return IGSrcs.size();
439 }
440 
441 /// Given a swizzle sequence SwzCandidate and an index Idx, returns the next
442 /// (in lexicographic term) swizzle sequence assuming that all swizzles after
443 /// Idx can be skipped
444 static bool
NextPossibleSolution(std::vector<R600InstrInfo::BankSwizzle> & SwzCandidate,unsigned Idx)445 NextPossibleSolution(
446     std::vector<R600InstrInfo::BankSwizzle> &SwzCandidate,
447     unsigned Idx) {
448   assert(Idx < SwzCandidate.size());
449   int ResetIdx = Idx;
450   while (ResetIdx > -1 && SwzCandidate[ResetIdx] == R600InstrInfo::ALU_VEC_210)
451     ResetIdx --;
452   for (unsigned i = ResetIdx + 1, e = SwzCandidate.size(); i < e; i++) {
453     SwzCandidate[i] = R600InstrInfo::ALU_VEC_012_SCL_210;
454   }
455   if (ResetIdx == -1)
456     return false;
457   int NextSwizzle = SwzCandidate[ResetIdx] + 1;
458   SwzCandidate[ResetIdx] = (R600InstrInfo::BankSwizzle)NextSwizzle;
459   return true;
460 }
461 
462 /// Enumerate all possible Swizzle sequence to find one that can meet all
463 /// read port requirements.
FindSwizzleForVectorSlot(const std::vector<std::vector<std::pair<int,unsigned>>> & IGSrcs,std::vector<R600InstrInfo::BankSwizzle> & SwzCandidate,const std::vector<std::pair<int,unsigned>> & TransSrcs,R600InstrInfo::BankSwizzle TransSwz) const464 bool R600InstrInfo::FindSwizzleForVectorSlot(
465     const std::vector<std::vector<std::pair<int, unsigned>>> &IGSrcs,
466     std::vector<R600InstrInfo::BankSwizzle> &SwzCandidate,
467     const std::vector<std::pair<int, unsigned>> &TransSrcs,
468     R600InstrInfo::BankSwizzle TransSwz) const {
469   unsigned ValidUpTo = 0;
470   do {
471     ValidUpTo = isLegalUpTo(IGSrcs, SwzCandidate, TransSrcs, TransSwz);
472     if (ValidUpTo == IGSrcs.size())
473       return true;
474   } while (NextPossibleSolution(SwzCandidate, ValidUpTo));
475   return false;
476 }
477 
478 /// Instructions in Trans slot can't read gpr at cycle 0 if they also read
479 /// a const, and can't read a gpr at cycle 1 if they read 2 const.
480 static bool
isConstCompatible(R600InstrInfo::BankSwizzle TransSwz,const std::vector<std::pair<int,unsigned>> & TransOps,unsigned ConstCount)481 isConstCompatible(R600InstrInfo::BankSwizzle TransSwz,
482                   const std::vector<std::pair<int, unsigned>> &TransOps,
483                   unsigned ConstCount) {
484   // TransALU can't read 3 constants
485   if (ConstCount > 2)
486     return false;
487   for (unsigned i = 0, e = TransOps.size(); i < e; ++i) {
488     const std::pair<int, unsigned> &Src = TransOps[i];
489     unsigned Cycle = getTransSwizzle(TransSwz, i);
490     if (Src.first < 0)
491       continue;
492     if (ConstCount > 0 && Cycle == 0)
493       return false;
494     if (ConstCount > 1 && Cycle == 1)
495       return false;
496   }
497   return true;
498 }
499 
500 bool
fitsReadPortLimitations(const std::vector<MachineInstr * > & IG,const DenseMap<unsigned,unsigned> & PV,std::vector<BankSwizzle> & ValidSwizzle,bool isLastAluTrans) const501 R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG,
502                                        const DenseMap<unsigned, unsigned> &PV,
503                                        std::vector<BankSwizzle> &ValidSwizzle,
504                                        bool isLastAluTrans)
505     const {
506   //Todo : support shared src0 - src1 operand
507 
508   std::vector<std::vector<std::pair<int, unsigned>>> IGSrcs;
509   ValidSwizzle.clear();
510   unsigned ConstCount;
511   BankSwizzle TransBS = ALU_VEC_012_SCL_210;
512   for (MachineInstr *MI : IG) {
513     IGSrcs.push_back(ExtractSrcs(*MI, PV, ConstCount));
514     unsigned Op = getOperandIdx(MI->getOpcode(), R600::OpName::bank_swizzle);
515     ValidSwizzle.push_back(
516         (R600InstrInfo::BankSwizzle)MI->getOperand(Op).getImm());
517   }
518   std::vector<std::pair<int, unsigned>> TransOps;
519   if (!isLastAluTrans)
520     return FindSwizzleForVectorSlot(IGSrcs, ValidSwizzle, TransOps, TransBS);
521 
522   TransOps = std::move(IGSrcs.back());
523   IGSrcs.pop_back();
524   ValidSwizzle.pop_back();
525 
526   static const R600InstrInfo::BankSwizzle TransSwz[] = {
527     ALU_VEC_012_SCL_210,
528     ALU_VEC_021_SCL_122,
529     ALU_VEC_120_SCL_212,
530     ALU_VEC_102_SCL_221
531   };
532   for (R600InstrInfo::BankSwizzle TransBS : TransSwz) {
533     if (!isConstCompatible(TransBS, TransOps, ConstCount))
534       continue;
535     bool Result = FindSwizzleForVectorSlot(IGSrcs, ValidSwizzle, TransOps,
536         TransBS);
537     if (Result) {
538       ValidSwizzle.push_back(TransBS);
539       return true;
540     }
541   }
542 
543   return false;
544 }
545 
546 bool
fitsConstReadLimitations(const std::vector<unsigned> & Consts) const547 R600InstrInfo::fitsConstReadLimitations(const std::vector<unsigned> &Consts)
548     const {
549   assert (Consts.size() <= 12 && "Too many operands in instructions group");
550   unsigned Pair1 = 0, Pair2 = 0;
551   for (unsigned Const : Consts) {
552     unsigned ReadConstHalf = Const & 2;
553     unsigned ReadConstIndex = Const & (~3);
554     unsigned ReadHalfConst = ReadConstIndex | ReadConstHalf;
555     if (!Pair1) {
556       Pair1 = ReadHalfConst;
557       continue;
558     }
559     if (Pair1 == ReadHalfConst)
560       continue;
561     if (!Pair2) {
562       Pair2 = ReadHalfConst;
563       continue;
564     }
565     if (Pair2 != ReadHalfConst)
566       return false;
567   }
568   return true;
569 }
570 
571 bool
fitsConstReadLimitations(const std::vector<MachineInstr * > & MIs) const572 R600InstrInfo::fitsConstReadLimitations(const std::vector<MachineInstr *> &MIs)
573     const {
574   std::vector<unsigned> Consts;
575   SmallSet<int64_t, 4> Literals;
576   for (MachineInstr *MI : MIs) {
577     if (!isALUInstr(MI->getOpcode()))
578       continue;
579 
580     for (const auto &Src : getSrcs(*MI)) {
581       if (Src.first->getReg() == R600::ALU_LITERAL_X)
582         Literals.insert(Src.second);
583       if (Literals.size() > 4)
584         return false;
585       if (Src.first->getReg() == R600::ALU_CONST)
586         Consts.push_back(Src.second);
587       if (R600::R600_KC0RegClass.contains(Src.first->getReg()) ||
588           R600::R600_KC1RegClass.contains(Src.first->getReg())) {
589         unsigned Index = RI.getEncodingValue(Src.first->getReg()) & 0xff;
590         unsigned Chan = RI.getHWRegChan(Src.first->getReg());
591         Consts.push_back((Index << 2) | Chan);
592       }
593     }
594   }
595   return fitsConstReadLimitations(Consts);
596 }
597 
598 DFAPacketizer *
CreateTargetScheduleState(const TargetSubtargetInfo & STI) const599 R600InstrInfo::CreateTargetScheduleState(const TargetSubtargetInfo &STI) const {
600   const InstrItineraryData *II = STI.getInstrItineraryData();
601   return static_cast<const R600Subtarget &>(STI).createDFAPacketizer(II);
602 }
603 
604 static bool
isPredicateSetter(unsigned Opcode)605 isPredicateSetter(unsigned Opcode) {
606   switch (Opcode) {
607   case R600::PRED_X:
608     return true;
609   default:
610     return false;
611   }
612 }
613 
614 static MachineInstr *
findFirstPredicateSetterFrom(MachineBasicBlock & MBB,MachineBasicBlock::iterator I)615 findFirstPredicateSetterFrom(MachineBasicBlock &MBB,
616                              MachineBasicBlock::iterator I) {
617   while (I != MBB.begin()) {
618     --I;
619     MachineInstr &MI = *I;
620     if (isPredicateSetter(MI.getOpcode()))
621       return &MI;
622   }
623 
624   return nullptr;
625 }
626 
627 static
isJump(unsigned Opcode)628 bool isJump(unsigned Opcode) {
629   return Opcode == R600::JUMP || Opcode == R600::JUMP_COND;
630 }
631 
isBranch(unsigned Opcode)632 static bool isBranch(unsigned Opcode) {
633   return Opcode == R600::BRANCH || Opcode == R600::BRANCH_COND_i32 ||
634       Opcode == R600::BRANCH_COND_f32;
635 }
636 
analyzeBranch(MachineBasicBlock & MBB,MachineBasicBlock * & TBB,MachineBasicBlock * & FBB,SmallVectorImpl<MachineOperand> & Cond,bool AllowModify) const637 bool R600InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
638                                   MachineBasicBlock *&TBB,
639                                   MachineBasicBlock *&FBB,
640                                   SmallVectorImpl<MachineOperand> &Cond,
641                                   bool AllowModify) const {
642   // Most of the following comes from the ARM implementation of analyzeBranch
643 
644   // If the block has no terminators, it just falls into the block after it.
645   MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
646   if (I == MBB.end())
647     return false;
648 
649   // R600::BRANCH* instructions are only available after isel and are not
650   // handled
651   if (isBranch(I->getOpcode()))
652     return true;
653   if (!isJump(I->getOpcode())) {
654     return false;
655   }
656 
657   // Remove successive JUMP
658   while (I != MBB.begin() && std::prev(I)->getOpcode() == R600::JUMP) {
659       MachineBasicBlock::iterator PriorI = std::prev(I);
660       if (AllowModify)
661         I->removeFromParent();
662       I = PriorI;
663   }
664   MachineInstr &LastInst = *I;
665 
666   // If there is only one terminator instruction, process it.
667   unsigned LastOpc = LastInst.getOpcode();
668   if (I == MBB.begin() || !isJump((--I)->getOpcode())) {
669     if (LastOpc == R600::JUMP) {
670       TBB = LastInst.getOperand(0).getMBB();
671       return false;
672     }
673     if (LastOpc == R600::JUMP_COND) {
674       auto predSet = I;
675       while (!isPredicateSetter(predSet->getOpcode())) {
676         predSet = --I;
677       }
678       TBB = LastInst.getOperand(0).getMBB();
679       Cond.push_back(predSet->getOperand(1));
680       Cond.push_back(predSet->getOperand(2));
681       Cond.push_back(MachineOperand::CreateReg(R600::PRED_SEL_ONE, false));
682       return false;
683     }
684     return true;  // Can't handle indirect branch.
685   }
686 
687   // Get the instruction before it if it is a terminator.
688   MachineInstr &SecondLastInst = *I;
689   unsigned SecondLastOpc = SecondLastInst.getOpcode();
690 
691   // If the block ends with a B and a Bcc, handle it.
692   if (SecondLastOpc == R600::JUMP_COND && LastOpc == R600::JUMP) {
693     auto predSet = --I;
694     while (!isPredicateSetter(predSet->getOpcode())) {
695       predSet = --I;
696     }
697     TBB = SecondLastInst.getOperand(0).getMBB();
698     FBB = LastInst.getOperand(0).getMBB();
699     Cond.push_back(predSet->getOperand(1));
700     Cond.push_back(predSet->getOperand(2));
701     Cond.push_back(MachineOperand::CreateReg(R600::PRED_SEL_ONE, false));
702     return false;
703   }
704 
705   // Otherwise, can't handle this.
706   return true;
707 }
708 
709 static
FindLastAluClause(MachineBasicBlock & MBB)710 MachineBasicBlock::iterator FindLastAluClause(MachineBasicBlock &MBB) {
711   for (MachineBasicBlock::reverse_iterator It = MBB.rbegin(), E = MBB.rend();
712       It != E; ++It) {
713     if (It->getOpcode() == R600::CF_ALU ||
714         It->getOpcode() == R600::CF_ALU_PUSH_BEFORE)
715       return It.getReverse();
716   }
717   return MBB.end();
718 }
719 
insertBranch(MachineBasicBlock & MBB,MachineBasicBlock * TBB,MachineBasicBlock * FBB,ArrayRef<MachineOperand> Cond,const DebugLoc & DL,int * BytesAdded) const720 unsigned R600InstrInfo::insertBranch(MachineBasicBlock &MBB,
721                                      MachineBasicBlock *TBB,
722                                      MachineBasicBlock *FBB,
723                                      ArrayRef<MachineOperand> Cond,
724                                      const DebugLoc &DL,
725                                      int *BytesAdded) const {
726   assert(TBB && "insertBranch must not be told to insert a fallthrough");
727   assert(!BytesAdded && "code size not handled");
728 
729   if (!FBB) {
730     if (Cond.empty()) {
731       BuildMI(&MBB, DL, get(R600::JUMP)).addMBB(TBB);
732       return 1;
733     }
734     MachineInstr *PredSet = findFirstPredicateSetterFrom(MBB, MBB.end());
735     assert(PredSet && "No previous predicate !");
736     addFlag(*PredSet, 0, MO_FLAG_PUSH);
737     PredSet->getOperand(2).setImm(Cond[1].getImm());
738 
739     BuildMI(&MBB, DL, get(R600::JUMP_COND))
740         .addMBB(TBB)
741         .addReg(R600::PREDICATE_BIT, RegState::Kill);
742     MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
743     if (CfAlu == MBB.end())
744       return 1;
745     assert (CfAlu->getOpcode() == R600::CF_ALU);
746     CfAlu->setDesc(get(R600::CF_ALU_PUSH_BEFORE));
747     return 1;
748   }
749   MachineInstr *PredSet = findFirstPredicateSetterFrom(MBB, MBB.end());
750   assert(PredSet && "No previous predicate !");
751   addFlag(*PredSet, 0, MO_FLAG_PUSH);
752   PredSet->getOperand(2).setImm(Cond[1].getImm());
753   BuildMI(&MBB, DL, get(R600::JUMP_COND))
754       .addMBB(TBB)
755       .addReg(R600::PREDICATE_BIT, RegState::Kill);
756   BuildMI(&MBB, DL, get(R600::JUMP)).addMBB(FBB);
757   MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
758   if (CfAlu == MBB.end())
759     return 2;
760   assert(CfAlu->getOpcode() == R600::CF_ALU);
761   CfAlu->setDesc(get(R600::CF_ALU_PUSH_BEFORE));
762   return 2;
763 }
764 
removeBranch(MachineBasicBlock & MBB,int * BytesRemoved) const765 unsigned R600InstrInfo::removeBranch(MachineBasicBlock &MBB,
766                                      int *BytesRemoved) const {
767   assert(!BytesRemoved && "code size not handled");
768 
769   // Note : we leave PRED* instructions there.
770   // They may be needed when predicating instructions.
771 
772   MachineBasicBlock::iterator I = MBB.end();
773 
774   if (I == MBB.begin()) {
775     return 0;
776   }
777   --I;
778   switch (I->getOpcode()) {
779   default:
780     return 0;
781   case R600::JUMP_COND: {
782     MachineInstr *predSet = findFirstPredicateSetterFrom(MBB, I);
783     clearFlag(*predSet, 0, MO_FLAG_PUSH);
784     I->eraseFromParent();
785     MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
786     if (CfAlu == MBB.end())
787       break;
788     assert (CfAlu->getOpcode() == R600::CF_ALU_PUSH_BEFORE);
789     CfAlu->setDesc(get(R600::CF_ALU));
790     break;
791   }
792   case R600::JUMP:
793     I->eraseFromParent();
794     break;
795   }
796   I = MBB.end();
797 
798   if (I == MBB.begin()) {
799     return 1;
800   }
801   --I;
802   switch (I->getOpcode()) {
803     // FIXME: only one case??
804   default:
805     return 1;
806   case R600::JUMP_COND: {
807     MachineInstr *predSet = findFirstPredicateSetterFrom(MBB, I);
808     clearFlag(*predSet, 0, MO_FLAG_PUSH);
809     I->eraseFromParent();
810     MachineBasicBlock::iterator CfAlu = FindLastAluClause(MBB);
811     if (CfAlu == MBB.end())
812       break;
813     assert (CfAlu->getOpcode() == R600::CF_ALU_PUSH_BEFORE);
814     CfAlu->setDesc(get(R600::CF_ALU));
815     break;
816   }
817   case R600::JUMP:
818     I->eraseFromParent();
819     break;
820   }
821   return 2;
822 }
823 
isPredicated(const MachineInstr & MI) const824 bool R600InstrInfo::isPredicated(const MachineInstr &MI) const {
825   int idx = MI.findFirstPredOperandIdx();
826   if (idx < 0)
827     return false;
828 
829   Register Reg = MI.getOperand(idx).getReg();
830   switch (Reg) {
831   default: return false;
832   case R600::PRED_SEL_ONE:
833   case R600::PRED_SEL_ZERO:
834   case R600::PREDICATE_BIT:
835     return true;
836   }
837 }
838 
isPredicable(const MachineInstr & MI) const839 bool R600InstrInfo::isPredicable(const MachineInstr &MI) const {
840   // XXX: KILL* instructions can be predicated, but they must be the last
841   // instruction in a clause, so this means any instructions after them cannot
842   // be predicated.  Until we have proper support for instruction clauses in the
843   // backend, we will mark KILL* instructions as unpredicable.
844 
845   if (MI.getOpcode() == R600::KILLGT)
846     return false;
847   if (MI.getOpcode() == R600::CF_ALU) {
848     // If the clause start in the middle of MBB then the MBB has more
849     // than a single clause, unable to predicate several clauses.
850     if (MI.getParent()->begin() != MachineBasicBlock::const_iterator(MI))
851       return false;
852     // TODO: We don't support KC merging atm
853     return MI.getOperand(3).getImm() == 0 && MI.getOperand(4).getImm() == 0;
854   }
855   if (isVector(MI))
856     return false;
857   return TargetInstrInfo::isPredicable(MI);
858 }
859 
860 bool
isProfitableToIfCvt(MachineBasicBlock & MBB,unsigned NumCycles,unsigned ExtraPredCycles,BranchProbability Probability) const861 R600InstrInfo::isProfitableToIfCvt(MachineBasicBlock &MBB,
862                                    unsigned NumCycles,
863                                    unsigned ExtraPredCycles,
864                                    BranchProbability Probability) const{
865   return true;
866 }
867 
868 bool
isProfitableToIfCvt(MachineBasicBlock & TMBB,unsigned NumTCycles,unsigned ExtraTCycles,MachineBasicBlock & FMBB,unsigned NumFCycles,unsigned ExtraFCycles,BranchProbability Probability) const869 R600InstrInfo::isProfitableToIfCvt(MachineBasicBlock &TMBB,
870                                    unsigned NumTCycles,
871                                    unsigned ExtraTCycles,
872                                    MachineBasicBlock &FMBB,
873                                    unsigned NumFCycles,
874                                    unsigned ExtraFCycles,
875                                    BranchProbability Probability) const {
876   return true;
877 }
878 
879 bool
isProfitableToDupForIfCvt(MachineBasicBlock & MBB,unsigned NumCycles,BranchProbability Probability) const880 R600InstrInfo::isProfitableToDupForIfCvt(MachineBasicBlock &MBB,
881                                          unsigned NumCycles,
882                                          BranchProbability Probability)
883                                          const {
884   return true;
885 }
886 
887 bool
isProfitableToUnpredicate(MachineBasicBlock & TMBB,MachineBasicBlock & FMBB) const888 R600InstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
889                                          MachineBasicBlock &FMBB) const {
890   return false;
891 }
892 
893 bool
reverseBranchCondition(SmallVectorImpl<MachineOperand> & Cond) const894 R600InstrInfo::reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const {
895   MachineOperand &MO = Cond[1];
896   switch (MO.getImm()) {
897   case R600::PRED_SETE_INT:
898     MO.setImm(R600::PRED_SETNE_INT);
899     break;
900   case R600::PRED_SETNE_INT:
901     MO.setImm(R600::PRED_SETE_INT);
902     break;
903   case R600::PRED_SETE:
904     MO.setImm(R600::PRED_SETNE);
905     break;
906   case R600::PRED_SETNE:
907     MO.setImm(R600::PRED_SETE);
908     break;
909   default:
910     return true;
911   }
912 
913   MachineOperand &MO2 = Cond[2];
914   switch (MO2.getReg()) {
915   case R600::PRED_SEL_ZERO:
916     MO2.setReg(R600::PRED_SEL_ONE);
917     break;
918   case R600::PRED_SEL_ONE:
919     MO2.setReg(R600::PRED_SEL_ZERO);
920     break;
921   default:
922     return true;
923   }
924   return false;
925 }
926 
ClobbersPredicate(MachineInstr & MI,std::vector<MachineOperand> & Pred,bool SkipDead) const927 bool R600InstrInfo::ClobbersPredicate(MachineInstr &MI,
928                                       std::vector<MachineOperand> &Pred,
929                                       bool SkipDead) const {
930   return isPredicateSetter(MI.getOpcode());
931 }
932 
PredicateInstruction(MachineInstr & MI,ArrayRef<MachineOperand> Pred) const933 bool R600InstrInfo::PredicateInstruction(MachineInstr &MI,
934                                          ArrayRef<MachineOperand> Pred) const {
935   int PIdx = MI.findFirstPredOperandIdx();
936 
937   if (MI.getOpcode() == R600::CF_ALU) {
938     MI.getOperand(8).setImm(0);
939     return true;
940   }
941 
942   if (MI.getOpcode() == R600::DOT_4) {
943     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_X))
944         .setReg(Pred[2].getReg());
945     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_Y))
946         .setReg(Pred[2].getReg());
947     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_Z))
948         .setReg(Pred[2].getReg());
949     MI.getOperand(getOperandIdx(MI, R600::OpName::pred_sel_W))
950         .setReg(Pred[2].getReg());
951     MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
952     MIB.addReg(R600::PREDICATE_BIT, RegState::Implicit);
953     return true;
954   }
955 
956   if (PIdx != -1) {
957     MachineOperand &PMO = MI.getOperand(PIdx);
958     PMO.setReg(Pred[2].getReg());
959     MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
960     MIB.addReg(R600::PREDICATE_BIT, RegState::Implicit);
961     return true;
962   }
963 
964   return false;
965 }
966 
getPredicationCost(const MachineInstr &) const967 unsigned int R600InstrInfo::getPredicationCost(const MachineInstr &) const {
968   return 2;
969 }
970 
getInstrLatency(const InstrItineraryData * ItinData,const MachineInstr &,unsigned * PredCost) const971 unsigned int R600InstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
972                                             const MachineInstr &,
973                                             unsigned *PredCost) const {
974   if (PredCost)
975     *PredCost = 2;
976   return 2;
977 }
978 
calculateIndirectAddress(unsigned RegIndex,unsigned Channel) const979 unsigned R600InstrInfo::calculateIndirectAddress(unsigned RegIndex,
980                                                    unsigned Channel) const {
981   assert(Channel == 0);
982   return RegIndex;
983 }
984 
expandPostRAPseudo(MachineInstr & MI) const985 bool R600InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
986   switch (MI.getOpcode()) {
987   default: {
988     MachineBasicBlock *MBB = MI.getParent();
989     int OffsetOpIdx =
990         R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::addr);
991     // addr is a custom operand with multiple MI operands, and only the
992     // first MI operand is given a name.
993     int RegOpIdx = OffsetOpIdx + 1;
994     int ChanOpIdx =
995         R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::chan);
996     if (isRegisterLoad(MI)) {
997       int DstOpIdx =
998           R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::dst);
999       unsigned RegIndex = MI.getOperand(RegOpIdx).getImm();
1000       unsigned Channel = MI.getOperand(ChanOpIdx).getImm();
1001       unsigned Address = calculateIndirectAddress(RegIndex, Channel);
1002       Register OffsetReg = MI.getOperand(OffsetOpIdx).getReg();
1003       if (OffsetReg == R600::INDIRECT_BASE_ADDR) {
1004         buildMovInstr(MBB, MI, MI.getOperand(DstOpIdx).getReg(),
1005                       getIndirectAddrRegClass()->getRegister(Address));
1006       } else {
1007         buildIndirectRead(MBB, MI, MI.getOperand(DstOpIdx).getReg(), Address,
1008                           OffsetReg);
1009       }
1010     } else if (isRegisterStore(MI)) {
1011       int ValOpIdx =
1012           R600::getNamedOperandIdx(MI.getOpcode(), R600::OpName::val);
1013       unsigned RegIndex = MI.getOperand(RegOpIdx).getImm();
1014       unsigned Channel = MI.getOperand(ChanOpIdx).getImm();
1015       unsigned Address = calculateIndirectAddress(RegIndex, Channel);
1016       Register OffsetReg = MI.getOperand(OffsetOpIdx).getReg();
1017       if (OffsetReg == R600::INDIRECT_BASE_ADDR) {
1018         buildMovInstr(MBB, MI, getIndirectAddrRegClass()->getRegister(Address),
1019                       MI.getOperand(ValOpIdx).getReg());
1020       } else {
1021         buildIndirectWrite(MBB, MI, MI.getOperand(ValOpIdx).getReg(),
1022                            calculateIndirectAddress(RegIndex, Channel),
1023                            OffsetReg);
1024       }
1025     } else {
1026       return false;
1027     }
1028 
1029     MBB->erase(MI);
1030     return true;
1031   }
1032   case R600::R600_EXTRACT_ELT_V2:
1033   case R600::R600_EXTRACT_ELT_V4:
1034     buildIndirectRead(MI.getParent(), MI, MI.getOperand(0).getReg(),
1035                       RI.getHWRegIndex(MI.getOperand(1).getReg()), //  Address
1036                       MI.getOperand(2).getReg(),
1037                       RI.getHWRegChan(MI.getOperand(1).getReg()));
1038     break;
1039   case R600::R600_INSERT_ELT_V2:
1040   case R600::R600_INSERT_ELT_V4:
1041     buildIndirectWrite(MI.getParent(), MI, MI.getOperand(2).getReg(), // Value
1042                        RI.getHWRegIndex(MI.getOperand(1).getReg()),   // Address
1043                        MI.getOperand(3).getReg(),                     // Offset
1044                        RI.getHWRegChan(MI.getOperand(1).getReg()));   // Channel
1045     break;
1046   }
1047   MI.eraseFromParent();
1048   return true;
1049 }
1050 
reserveIndirectRegisters(BitVector & Reserved,const MachineFunction & MF,const R600RegisterInfo & TRI) const1051 void R600InstrInfo::reserveIndirectRegisters(BitVector &Reserved,
1052                                              const MachineFunction &MF,
1053                                              const R600RegisterInfo &TRI) const {
1054   const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
1055   const R600FrameLowering *TFL = ST.getFrameLowering();
1056 
1057   unsigned StackWidth = TFL->getStackWidth(MF);
1058   int End = getIndirectIndexEnd(MF);
1059 
1060   if (End == -1)
1061     return;
1062 
1063   for (int Index = getIndirectIndexBegin(MF); Index <= End; ++Index) {
1064     for (unsigned Chan = 0; Chan < StackWidth; ++Chan) {
1065       MCRegister Reg =
1066           R600::R600_TReg32RegClass.getRegister((4 * Index) + Chan);
1067       TRI.reserveRegisterTuples(Reserved, Reg);
1068     }
1069   }
1070 }
1071 
getIndirectAddrRegClass() const1072 const TargetRegisterClass *R600InstrInfo::getIndirectAddrRegClass() const {
1073   return &R600::R600_TReg32_XRegClass;
1074 }
1075 
buildIndirectWrite(MachineBasicBlock * MBB,MachineBasicBlock::iterator I,unsigned ValueReg,unsigned Address,unsigned OffsetReg) const1076 MachineInstrBuilder R600InstrInfo::buildIndirectWrite(MachineBasicBlock *MBB,
1077                                        MachineBasicBlock::iterator I,
1078                                        unsigned ValueReg, unsigned Address,
1079                                        unsigned OffsetReg) const {
1080   return buildIndirectWrite(MBB, I, ValueReg, Address, OffsetReg, 0);
1081 }
1082 
buildIndirectWrite(MachineBasicBlock * MBB,MachineBasicBlock::iterator I,unsigned ValueReg,unsigned Address,unsigned OffsetReg,unsigned AddrChan) const1083 MachineInstrBuilder R600InstrInfo::buildIndirectWrite(MachineBasicBlock *MBB,
1084                                        MachineBasicBlock::iterator I,
1085                                        unsigned ValueReg, unsigned Address,
1086                                        unsigned OffsetReg,
1087                                        unsigned AddrChan) const {
1088   MCRegister AddrReg;
1089   switch (AddrChan) {
1090     default: llvm_unreachable("Invalid Channel");
1091     case 0: AddrReg = R600::R600_AddrRegClass.getRegister(Address); break;
1092     case 1: AddrReg = R600::R600_Addr_YRegClass.getRegister(Address); break;
1093     case 2: AddrReg = R600::R600_Addr_ZRegClass.getRegister(Address); break;
1094     case 3: AddrReg = R600::R600_Addr_WRegClass.getRegister(Address); break;
1095   }
1096   MachineInstr *MOVA = buildDefaultInstruction(*MBB, I, R600::MOVA_INT_eg,
1097                                                R600::AR_X, OffsetReg);
1098   setImmOperand(*MOVA, R600::OpName::write, 0);
1099 
1100   MachineInstrBuilder Mov = buildDefaultInstruction(*MBB, I, R600::MOV,
1101                                       AddrReg, ValueReg)
1102                                       .addReg(R600::AR_X,
1103                                            RegState::Implicit | RegState::Kill);
1104   setImmOperand(*Mov, R600::OpName::dst_rel, 1);
1105   return Mov;
1106 }
1107 
buildIndirectRead(MachineBasicBlock * MBB,MachineBasicBlock::iterator I,unsigned ValueReg,unsigned Address,unsigned OffsetReg) const1108 MachineInstrBuilder R600InstrInfo::buildIndirectRead(MachineBasicBlock *MBB,
1109                                        MachineBasicBlock::iterator I,
1110                                        unsigned ValueReg, unsigned Address,
1111                                        unsigned OffsetReg) const {
1112   return buildIndirectRead(MBB, I, ValueReg, Address, OffsetReg, 0);
1113 }
1114 
buildIndirectRead(MachineBasicBlock * MBB,MachineBasicBlock::iterator I,unsigned ValueReg,unsigned Address,unsigned OffsetReg,unsigned AddrChan) const1115 MachineInstrBuilder R600InstrInfo::buildIndirectRead(MachineBasicBlock *MBB,
1116                                        MachineBasicBlock::iterator I,
1117                                        unsigned ValueReg, unsigned Address,
1118                                        unsigned OffsetReg,
1119                                        unsigned AddrChan) const {
1120   MCRegister AddrReg;
1121   switch (AddrChan) {
1122     default: llvm_unreachable("Invalid Channel");
1123     case 0: AddrReg = R600::R600_AddrRegClass.getRegister(Address); break;
1124     case 1: AddrReg = R600::R600_Addr_YRegClass.getRegister(Address); break;
1125     case 2: AddrReg = R600::R600_Addr_ZRegClass.getRegister(Address); break;
1126     case 3: AddrReg = R600::R600_Addr_WRegClass.getRegister(Address); break;
1127   }
1128   MachineInstr *MOVA = buildDefaultInstruction(*MBB, I, R600::MOVA_INT_eg,
1129                                                        R600::AR_X,
1130                                                        OffsetReg);
1131   setImmOperand(*MOVA, R600::OpName::write, 0);
1132   MachineInstrBuilder Mov = buildDefaultInstruction(*MBB, I, R600::MOV,
1133                                       ValueReg,
1134                                       AddrReg)
1135                                       .addReg(R600::AR_X,
1136                                            RegState::Implicit | RegState::Kill);
1137   setImmOperand(*Mov, R600::OpName::src0_rel, 1);
1138 
1139   return Mov;
1140 }
1141 
getIndirectIndexBegin(const MachineFunction & MF) const1142 int R600InstrInfo::getIndirectIndexBegin(const MachineFunction &MF) const {
1143   const MachineRegisterInfo &MRI = MF.getRegInfo();
1144   const MachineFrameInfo &MFI = MF.getFrameInfo();
1145   int Offset = -1;
1146 
1147   if (MFI.getNumObjects() == 0) {
1148     return -1;
1149   }
1150 
1151   if (MRI.livein_empty()) {
1152     return 0;
1153   }
1154 
1155   const TargetRegisterClass *IndirectRC = getIndirectAddrRegClass();
1156   for (std::pair<MCRegister, Register> LI : MRI.liveins()) {
1157     Register Reg = LI.first;
1158     if (Reg.isVirtual() || !IndirectRC->contains(Reg))
1159       continue;
1160 
1161     unsigned RegIndex;
1162     unsigned RegEnd;
1163     for (RegIndex = 0, RegEnd = IndirectRC->getNumRegs(); RegIndex != RegEnd;
1164                                                           ++RegIndex) {
1165       if (IndirectRC->getRegister(RegIndex) == (unsigned)Reg)
1166         break;
1167     }
1168     Offset = std::max(Offset, (int)RegIndex);
1169   }
1170 
1171   return Offset + 1;
1172 }
1173 
getIndirectIndexEnd(const MachineFunction & MF) const1174 int R600InstrInfo::getIndirectIndexEnd(const MachineFunction &MF) const {
1175   int Offset = 0;
1176   const MachineFrameInfo &MFI = MF.getFrameInfo();
1177 
1178   // Variable sized objects are not supported
1179   if (MFI.hasVarSizedObjects()) {
1180     return -1;
1181   }
1182 
1183   if (MFI.getNumObjects() == 0) {
1184     return -1;
1185   }
1186 
1187   const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
1188   const R600FrameLowering *TFL = ST.getFrameLowering();
1189 
1190   Register IgnoredFrameReg;
1191   Offset = TFL->getFrameIndexReference(MF, -1, IgnoredFrameReg).getFixed();
1192 
1193   return getIndirectIndexBegin(MF) + Offset;
1194 }
1195 
getMaxAlusPerClause() const1196 unsigned R600InstrInfo::getMaxAlusPerClause() const {
1197   return 115;
1198 }
1199 
buildDefaultInstruction(MachineBasicBlock & MBB,MachineBasicBlock::iterator I,unsigned Opcode,unsigned DstReg,unsigned Src0Reg,unsigned Src1Reg) const1200 MachineInstrBuilder R600InstrInfo::buildDefaultInstruction(MachineBasicBlock &MBB,
1201                                                   MachineBasicBlock::iterator I,
1202                                                   unsigned Opcode,
1203                                                   unsigned DstReg,
1204                                                   unsigned Src0Reg,
1205                                                   unsigned Src1Reg) const {
1206   MachineInstrBuilder MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opcode),
1207     DstReg);           // $dst
1208 
1209   if (Src1Reg) {
1210     MIB.addImm(0)     // $update_exec_mask
1211        .addImm(0);    // $update_predicate
1212   }
1213   MIB.addImm(1)        // $write
1214      .addImm(0)        // $omod
1215      .addImm(0)        // $dst_rel
1216      .addImm(0)        // $dst_clamp
1217      .addReg(Src0Reg)  // $src0
1218      .addImm(0)        // $src0_neg
1219      .addImm(0)        // $src0_rel
1220      .addImm(0)        // $src0_abs
1221      .addImm(-1);       // $src0_sel
1222 
1223   if (Src1Reg) {
1224     MIB.addReg(Src1Reg) // $src1
1225        .addImm(0)       // $src1_neg
1226        .addImm(0)       // $src1_rel
1227        .addImm(0)       // $src1_abs
1228        .addImm(-1);      // $src1_sel
1229   }
1230 
1231   //XXX: The r600g finalizer expects this to be 1, once we've moved the
1232   //scheduling to the backend, we can change the default to 0.
1233   MIB.addImm(1)        // $last
1234       .addReg(R600::PRED_SEL_OFF) // $pred_sel
1235       .addImm(0)         // $literal
1236       .addImm(0);        // $bank_swizzle
1237 
1238   return MIB;
1239 }
1240 
1241 #define OPERAND_CASE(Label)                                                    \
1242   case Label: {                                                                \
1243     static const R600::OpName Ops[] = {Label##_X, Label##_Y, Label##_Z,        \
1244                                        Label##_W};                             \
1245     return Ops[Slot];                                                          \
1246   }
1247 
getSlotedOps(R600::OpName Op,unsigned Slot)1248 static R600::OpName getSlotedOps(R600::OpName Op, unsigned Slot) {
1249   switch (Op) {
1250   OPERAND_CASE(R600::OpName::update_exec_mask)
1251   OPERAND_CASE(R600::OpName::update_pred)
1252   OPERAND_CASE(R600::OpName::write)
1253   OPERAND_CASE(R600::OpName::omod)
1254   OPERAND_CASE(R600::OpName::dst_rel)
1255   OPERAND_CASE(R600::OpName::clamp)
1256   OPERAND_CASE(R600::OpName::src0)
1257   OPERAND_CASE(R600::OpName::src0_neg)
1258   OPERAND_CASE(R600::OpName::src0_rel)
1259   OPERAND_CASE(R600::OpName::src0_abs)
1260   OPERAND_CASE(R600::OpName::src0_sel)
1261   OPERAND_CASE(R600::OpName::src1)
1262   OPERAND_CASE(R600::OpName::src1_neg)
1263   OPERAND_CASE(R600::OpName::src1_rel)
1264   OPERAND_CASE(R600::OpName::src1_abs)
1265   OPERAND_CASE(R600::OpName::src1_sel)
1266   OPERAND_CASE(R600::OpName::pred_sel)
1267   default:
1268     llvm_unreachable("Wrong Operand");
1269   }
1270 }
1271 
1272 #undef OPERAND_CASE
1273 
buildSlotOfVectorInstruction(MachineBasicBlock & MBB,MachineInstr * MI,unsigned Slot,unsigned DstReg) const1274 MachineInstr *R600InstrInfo::buildSlotOfVectorInstruction(
1275     MachineBasicBlock &MBB, MachineInstr *MI, unsigned Slot, unsigned DstReg)
1276     const {
1277   assert (MI->getOpcode() == R600::DOT_4 && "Not Implemented");
1278   unsigned Opcode;
1279   if (ST.getGeneration() <= AMDGPUSubtarget::R700)
1280     Opcode = R600::DOT4_r600;
1281   else
1282     Opcode = R600::DOT4_eg;
1283   MachineBasicBlock::iterator I = MI;
1284   MachineOperand &Src0 = MI->getOperand(
1285       getOperandIdx(MI->getOpcode(), getSlotedOps(R600::OpName::src0, Slot)));
1286   MachineOperand &Src1 = MI->getOperand(
1287       getOperandIdx(MI->getOpcode(), getSlotedOps(R600::OpName::src1, Slot)));
1288   MachineInstr *MIB = buildDefaultInstruction(
1289       MBB, I, Opcode, DstReg, Src0.getReg(), Src1.getReg());
1290   static const R600::OpName Operands[14] = {
1291       R600::OpName::update_exec_mask,
1292       R600::OpName::update_pred,
1293       R600::OpName::write,
1294       R600::OpName::omod,
1295       R600::OpName::dst_rel,
1296       R600::OpName::clamp,
1297       R600::OpName::src0_neg,
1298       R600::OpName::src0_rel,
1299       R600::OpName::src0_abs,
1300       R600::OpName::src0_sel,
1301       R600::OpName::src1_neg,
1302       R600::OpName::src1_rel,
1303       R600::OpName::src1_abs,
1304       R600::OpName::src1_sel,
1305   };
1306 
1307   MachineOperand &MO = MI->getOperand(getOperandIdx(MI->getOpcode(),
1308       getSlotedOps(R600::OpName::pred_sel, Slot)));
1309   MIB->getOperand(getOperandIdx(Opcode, R600::OpName::pred_sel))
1310       .setReg(MO.getReg());
1311 
1312   for (R600::OpName Operand : Operands) {
1313     MachineOperand &MO = MI->getOperand(
1314         getOperandIdx(MI->getOpcode(), getSlotedOps(Operand, Slot)));
1315     assert (MO.isImm());
1316     setImmOperand(*MIB, Operand, MO.getImm());
1317   }
1318   MIB->getOperand(20).setImm(0);
1319   return MIB;
1320 }
1321 
buildMovImm(MachineBasicBlock & BB,MachineBasicBlock::iterator I,unsigned DstReg,uint64_t Imm) const1322 MachineInstr *R600InstrInfo::buildMovImm(MachineBasicBlock &BB,
1323                                          MachineBasicBlock::iterator I,
1324                                          unsigned DstReg,
1325                                          uint64_t Imm) const {
1326   MachineInstr *MovImm = buildDefaultInstruction(BB, I, R600::MOV, DstReg,
1327                                                   R600::ALU_LITERAL_X);
1328   setImmOperand(*MovImm, R600::OpName::literal, Imm);
1329   return MovImm;
1330 }
1331 
buildMovInstr(MachineBasicBlock * MBB,MachineBasicBlock::iterator I,unsigned DstReg,unsigned SrcReg) const1332 MachineInstr *R600InstrInfo::buildMovInstr(MachineBasicBlock *MBB,
1333                                        MachineBasicBlock::iterator I,
1334                                        unsigned DstReg, unsigned SrcReg) const {
1335   return buildDefaultInstruction(*MBB, I, R600::MOV, DstReg, SrcReg);
1336 }
1337 
getOperandIdx(const MachineInstr & MI,R600::OpName Op) const1338 int R600InstrInfo::getOperandIdx(const MachineInstr &MI,
1339                                  R600::OpName Op) const {
1340   return getOperandIdx(MI.getOpcode(), Op);
1341 }
1342 
getOperandIdx(unsigned Opcode,R600::OpName Op) const1343 int R600InstrInfo::getOperandIdx(unsigned Opcode, R600::OpName Op) const {
1344   return R600::getNamedOperandIdx(Opcode, Op);
1345 }
1346 
setImmOperand(MachineInstr & MI,R600::OpName Op,int64_t Imm) const1347 void R600InstrInfo::setImmOperand(MachineInstr &MI, R600::OpName Op,
1348                                   int64_t Imm) const {
1349   int Idx = getOperandIdx(MI, Op);
1350   assert(Idx != -1 && "Operand not supported for this instruction.");
1351   assert(MI.getOperand(Idx).isImm());
1352   MI.getOperand(Idx).setImm(Imm);
1353 }
1354 
1355 //===----------------------------------------------------------------------===//
1356 // Instruction flag getters/setters
1357 //===----------------------------------------------------------------------===//
1358 
getFlagOp(MachineInstr & MI,unsigned SrcIdx,unsigned Flag) const1359 MachineOperand &R600InstrInfo::getFlagOp(MachineInstr &MI, unsigned SrcIdx,
1360                                          unsigned Flag) const {
1361   unsigned TargetFlags = get(MI.getOpcode()).TSFlags;
1362   int FlagIndex = 0;
1363   if (Flag != 0) {
1364     // If we pass something other than the default value of Flag to this
1365     // function, it means we are want to set a flag on an instruction
1366     // that uses native encoding.
1367     assert(HAS_NATIVE_OPERANDS(TargetFlags));
1368     bool IsOP3 = (TargetFlags & R600_InstFlag::OP3) == R600_InstFlag::OP3;
1369     switch (Flag) {
1370     case MO_FLAG_CLAMP:
1371       FlagIndex = getOperandIdx(MI, R600::OpName::clamp);
1372       break;
1373     case MO_FLAG_MASK:
1374       FlagIndex = getOperandIdx(MI, R600::OpName::write);
1375       break;
1376     case MO_FLAG_NOT_LAST:
1377     case MO_FLAG_LAST:
1378       FlagIndex = getOperandIdx(MI, R600::OpName::last);
1379       break;
1380     case MO_FLAG_NEG:
1381       switch (SrcIdx) {
1382       case 0:
1383         FlagIndex = getOperandIdx(MI, R600::OpName::src0_neg);
1384         break;
1385       case 1:
1386         FlagIndex = getOperandIdx(MI, R600::OpName::src1_neg);
1387         break;
1388       case 2:
1389         FlagIndex = getOperandIdx(MI, R600::OpName::src2_neg);
1390         break;
1391       }
1392       break;
1393 
1394     case MO_FLAG_ABS:
1395       assert(!IsOP3 && "Cannot set absolute value modifier for OP3 "
1396                        "instructions.");
1397       (void)IsOP3;
1398       switch (SrcIdx) {
1399       case 0:
1400         FlagIndex = getOperandIdx(MI, R600::OpName::src0_abs);
1401         break;
1402       case 1:
1403         FlagIndex = getOperandIdx(MI, R600::OpName::src1_abs);
1404         break;
1405       }
1406       break;
1407 
1408     default:
1409       FlagIndex = -1;
1410       break;
1411     }
1412     assert(FlagIndex != -1 && "Flag not supported for this instruction");
1413   } else {
1414       FlagIndex = GET_FLAG_OPERAND_IDX(TargetFlags);
1415       assert(FlagIndex != 0 &&
1416          "Instruction flags not supported for this instruction");
1417   }
1418 
1419   MachineOperand &FlagOp = MI.getOperand(FlagIndex);
1420   assert(FlagOp.isImm());
1421   return FlagOp;
1422 }
1423 
addFlag(MachineInstr & MI,unsigned SrcIdx,unsigned Flag) const1424 void R600InstrInfo::addFlag(MachineInstr &MI, unsigned SrcIdx,
1425                             unsigned Flag) const {
1426   unsigned TargetFlags = get(MI.getOpcode()).TSFlags;
1427   if (Flag == 0) {
1428     return;
1429   }
1430   if (HAS_NATIVE_OPERANDS(TargetFlags)) {
1431     MachineOperand &FlagOp = getFlagOp(MI, SrcIdx, Flag);
1432     if (Flag == MO_FLAG_NOT_LAST) {
1433       clearFlag(MI, SrcIdx, MO_FLAG_LAST);
1434     } else if (Flag == MO_FLAG_MASK) {
1435       clearFlag(MI, SrcIdx, Flag);
1436     } else {
1437       FlagOp.setImm(1);
1438     }
1439   } else {
1440     MachineOperand &FlagOp = getFlagOp(MI, SrcIdx);
1441     FlagOp.setImm(FlagOp.getImm() | (Flag << (NUM_MO_FLAGS * SrcIdx)));
1442   }
1443 }
1444 
clearFlag(MachineInstr & MI,unsigned SrcIdx,unsigned Flag) const1445 void R600InstrInfo::clearFlag(MachineInstr &MI, unsigned SrcIdx,
1446                               unsigned Flag) const {
1447   unsigned TargetFlags = get(MI.getOpcode()).TSFlags;
1448   if (HAS_NATIVE_OPERANDS(TargetFlags)) {
1449     MachineOperand &FlagOp = getFlagOp(MI, SrcIdx, Flag);
1450     FlagOp.setImm(0);
1451   } else {
1452     MachineOperand &FlagOp = getFlagOp(MI);
1453     unsigned InstFlags = FlagOp.getImm();
1454     InstFlags &= ~(Flag << (NUM_MO_FLAGS * SrcIdx));
1455     FlagOp.setImm(InstFlags);
1456   }
1457 }
1458