xref: /freebsd/contrib/llvm-project/llvm/lib/Target/AMDGPU/R600Packetizer.cpp (revision 5956d97f4b3204318ceb6aa9c77bd0bc6ea87a41) 
1 //===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass implements instructions packetization for R600. It unsets isLast
11 /// bit of instructions inside a bundle and substitutes src register with
12 /// PreviousVector when applicable.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "MCTargetDesc/R600MCTargetDesc.h"
17 #include "R600.h"
18 #include "R600Subtarget.h"
19 #include "llvm/CodeGen/DFAPacketizer.h"
20 #include "llvm/CodeGen/MachineDominators.h"
21 #include "llvm/CodeGen/MachineLoopInfo.h"
22 #include "llvm/CodeGen/ScheduleDAG.h"
23 
24 using namespace llvm;
25 
26 #define DEBUG_TYPE "packets"
27 
28 namespace {
29 
30 class R600Packetizer : public MachineFunctionPass {
31 
32 public:
33   static char ID;
34   R600Packetizer() : MachineFunctionPass(ID) {}
35 
36   void getAnalysisUsage(AnalysisUsage &AU) const override {
37     AU.setPreservesCFG();
38     AU.addRequired<MachineDominatorTree>();
39     AU.addPreserved<MachineDominatorTree>();
40     AU.addRequired<MachineLoopInfo>();
41     AU.addPreserved<MachineLoopInfo>();
42     MachineFunctionPass::getAnalysisUsage(AU);
43   }
44 
45   StringRef getPassName() const override { return "R600 Packetizer"; }
46 
47   bool runOnMachineFunction(MachineFunction &Fn) override;
48 };
49 
50 class R600PacketizerList : public VLIWPacketizerList {
51 private:
52   const R600InstrInfo *TII;
53   const R600RegisterInfo &TRI;
54   bool VLIW5;
55   bool ConsideredInstUsesAlreadyWrittenVectorElement;
56 
57   unsigned getSlot(const MachineInstr &MI) const {
58     return TRI.getHWRegChan(MI.getOperand(0).getReg());
59   }
60 
61   /// \returns register to PV chan mapping for bundle/single instructions that
62   /// immediately precedes I.
63   DenseMap<unsigned, unsigned> getPreviousVector(MachineBasicBlock::iterator I)
64       const {
65     DenseMap<unsigned, unsigned> Result;
66     I--;
67     if (!TII->isALUInstr(I->getOpcode()) && !I->isBundle())
68       return Result;
69     MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
70     if (I->isBundle())
71       BI++;
72     int LastDstChan = -1;
73     do {
74       bool isTrans = false;
75       int BISlot = getSlot(*BI);
76       if (LastDstChan >= BISlot)
77         isTrans = true;
78       LastDstChan = BISlot;
79       if (TII->isPredicated(*BI))
80         continue;
81       int OperandIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::write);
82       if (OperandIdx > -1 && BI->getOperand(OperandIdx).getImm() == 0)
83         continue;
84       int DstIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::dst);
85       if (DstIdx == -1) {
86         continue;
87       }
88       Register Dst = BI->getOperand(DstIdx).getReg();
89       if (isTrans || TII->isTransOnly(*BI)) {
90         Result[Dst] = R600::PS;
91         continue;
92       }
93       if (BI->getOpcode() == R600::DOT4_r600 ||
94           BI->getOpcode() == R600::DOT4_eg) {
95         Result[Dst] = R600::PV_X;
96         continue;
97       }
98       if (Dst == R600::OQAP) {
99         continue;
100       }
101       unsigned PVReg = 0;
102       switch (TRI.getHWRegChan(Dst)) {
103       case 0:
104         PVReg = R600::PV_X;
105         break;
106       case 1:
107         PVReg = R600::PV_Y;
108         break;
109       case 2:
110         PVReg = R600::PV_Z;
111         break;
112       case 3:
113         PVReg = R600::PV_W;
114         break;
115       default:
116         llvm_unreachable("Invalid Chan");
117       }
118       Result[Dst] = PVReg;
119     } while ((++BI)->isBundledWithPred());
120     return Result;
121   }
122 
123   void substitutePV(MachineInstr &MI, const DenseMap<unsigned, unsigned> &PVs)
124       const {
125     unsigned Ops[] = {
126       R600::OpName::src0,
127       R600::OpName::src1,
128       R600::OpName::src2
129     };
130     for (unsigned Op : Ops) {
131       int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Op);
132       if (OperandIdx < 0)
133         continue;
134       Register Src = MI.getOperand(OperandIdx).getReg();
135       const DenseMap<unsigned, unsigned>::const_iterator It = PVs.find(Src);
136       if (It != PVs.end())
137         MI.getOperand(OperandIdx).setReg(It->second);
138     }
139   }
140 public:
141   // Ctor.
142   R600PacketizerList(MachineFunction &MF, const R600Subtarget &ST,
143                      MachineLoopInfo &MLI)
144       : VLIWPacketizerList(MF, MLI, nullptr),
145         TII(ST.getInstrInfo()),
146         TRI(TII->getRegisterInfo()) {
147     VLIW5 = !ST.hasCaymanISA();
148   }
149 
150   // initPacketizerState - initialize some internal flags.
151   void initPacketizerState() override {
152     ConsideredInstUsesAlreadyWrittenVectorElement = false;
153   }
154 
155   // ignorePseudoInstruction - Ignore bundling of pseudo instructions.
156   bool ignorePseudoInstruction(const MachineInstr &MI,
157                                const MachineBasicBlock *MBB) override {
158     return false;
159   }
160 
161   // isSoloInstruction - return true if instruction MI can not be packetized
162   // with any other instruction, which means that MI itself is a packet.
163   bool isSoloInstruction(const MachineInstr &MI) override {
164     if (TII->isVector(MI))
165       return true;
166     if (!TII->isALUInstr(MI.getOpcode()))
167       return true;
168     if (MI.getOpcode() == R600::GROUP_BARRIER)
169       return true;
170     // XXX: This can be removed once the packetizer properly handles all the
171     // LDS instruction group restrictions.
172     return TII->isLDSInstr(MI.getOpcode());
173   }
174 
175   // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ
176   // together.
177   bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override {
178     MachineInstr *MII = SUI->getInstr(), *MIJ = SUJ->getInstr();
179     if (getSlot(*MII) == getSlot(*MIJ))
180       ConsideredInstUsesAlreadyWrittenVectorElement = true;
181     // Does MII and MIJ share the same pred_sel ?
182     int OpI = TII->getOperandIdx(MII->getOpcode(), R600::OpName::pred_sel),
183         OpJ = TII->getOperandIdx(MIJ->getOpcode(), R600::OpName::pred_sel);
184     Register PredI = (OpI > -1)?MII->getOperand(OpI).getReg() : Register(),
185       PredJ = (OpJ > -1)?MIJ->getOperand(OpJ).getReg() : Register();
186     if (PredI != PredJ)
187       return false;
188     if (SUJ->isSucc(SUI)) {
189       for (unsigned i = 0, e = SUJ->Succs.size(); i < e; ++i) {
190         const SDep &Dep = SUJ->Succs[i];
191         if (Dep.getSUnit() != SUI)
192           continue;
193         if (Dep.getKind() == SDep::Anti)
194           continue;
195         if (Dep.getKind() == SDep::Output)
196           if (MII->getOperand(0).getReg() != MIJ->getOperand(0).getReg())
197             continue;
198         return false;
199       }
200     }
201 
202     bool ARDef =
203         TII->definesAddressRegister(*MII) || TII->definesAddressRegister(*MIJ);
204     bool ARUse =
205         TII->usesAddressRegister(*MII) || TII->usesAddressRegister(*MIJ);
206 
207     return !ARDef || !ARUse;
208   }
209 
210   // isLegalToPruneDependencies - Is it legal to prune dependece between SUI
211   // and SUJ.
212   bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override {
213     return false;
214   }
215 
216   void setIsLastBit(MachineInstr *MI, unsigned Bit) const {
217     unsigned LastOp = TII->getOperandIdx(MI->getOpcode(), R600::OpName::last);
218     MI->getOperand(LastOp).setImm(Bit);
219   }
220 
221   bool isBundlableWithCurrentPMI(MachineInstr &MI,
222                                  const DenseMap<unsigned, unsigned> &PV,
223                                  std::vector<R600InstrInfo::BankSwizzle> &BS,
224                                  bool &isTransSlot) {
225     isTransSlot = TII->isTransOnly(MI);
226     assert (!isTransSlot || VLIW5);
227 
228     // Is the dst reg sequence legal ?
229     if (!isTransSlot && !CurrentPacketMIs.empty()) {
230       if (getSlot(MI) <= getSlot(*CurrentPacketMIs.back())) {
231         if (ConsideredInstUsesAlreadyWrittenVectorElement &&
232             !TII->isVectorOnly(MI) && VLIW5) {
233           isTransSlot = true;
234           LLVM_DEBUG({
235             dbgs() << "Considering as Trans Inst :";
236             MI.dump();
237           });
238         }
239         else
240           return false;
241       }
242     }
243 
244     // Are the Constants limitations met ?
245     CurrentPacketMIs.push_back(&MI);
246     if (!TII->fitsConstReadLimitations(CurrentPacketMIs)) {
247       LLVM_DEBUG({
248         dbgs() << "Couldn't pack :\n";
249         MI.dump();
250         dbgs() << "with the following packets :\n";
251         for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) {
252           CurrentPacketMIs[i]->dump();
253           dbgs() << "\n";
254         }
255         dbgs() << "because of Consts read limitations\n";
256       });
257       CurrentPacketMIs.pop_back();
258       return false;
259     }
260 
261     // Is there a BankSwizzle set that meet Read Port limitations ?
262     if (!TII->fitsReadPortLimitations(CurrentPacketMIs,
263             PV, BS, isTransSlot)) {
264       LLVM_DEBUG({
265         dbgs() << "Couldn't pack :\n";
266         MI.dump();
267         dbgs() << "with the following packets :\n";
268         for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) {
269           CurrentPacketMIs[i]->dump();
270           dbgs() << "\n";
271         }
272         dbgs() << "because of Read port limitations\n";
273       });
274       CurrentPacketMIs.pop_back();
275       return false;
276     }
277 
278     // We cannot read LDS source registers from the Trans slot.
279     if (isTransSlot && TII->readsLDSSrcReg(MI))
280       return false;
281 
282     CurrentPacketMIs.pop_back();
283     return true;
284   }
285 
286   MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override {
287     MachineBasicBlock::iterator FirstInBundle =
288         CurrentPacketMIs.empty() ? &MI : CurrentPacketMIs.front();
289     const DenseMap<unsigned, unsigned> &PV =
290         getPreviousVector(FirstInBundle);
291     std::vector<R600InstrInfo::BankSwizzle> BS;
292     bool isTransSlot;
293 
294     if (isBundlableWithCurrentPMI(MI, PV, BS, isTransSlot)) {
295       for (unsigned i = 0, e = CurrentPacketMIs.size(); i < e; i++) {
296         MachineInstr *MI = CurrentPacketMIs[i];
297         unsigned Op = TII->getOperandIdx(MI->getOpcode(),
298             R600::OpName::bank_swizzle);
299         MI->getOperand(Op).setImm(BS[i]);
300       }
301       unsigned Op =
302           TII->getOperandIdx(MI.getOpcode(), R600::OpName::bank_swizzle);
303       MI.getOperand(Op).setImm(BS.back());
304       if (!CurrentPacketMIs.empty())
305         setIsLastBit(CurrentPacketMIs.back(), 0);
306       substitutePV(MI, PV);
307       MachineBasicBlock::iterator It = VLIWPacketizerList::addToPacket(MI);
308       if (isTransSlot) {
309         endPacket(std::next(It)->getParent(), std::next(It));
310       }
311       return It;
312     }
313     endPacket(MI.getParent(), MI);
314     if (TII->isTransOnly(MI))
315       return MI;
316     return VLIWPacketizerList::addToPacket(MI);
317   }
318 };
319 
320 bool R600Packetizer::runOnMachineFunction(MachineFunction &Fn) {
321   const R600Subtarget &ST = Fn.getSubtarget<R600Subtarget>();
322   const R600InstrInfo *TII = ST.getInstrInfo();
323 
324   MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>();
325 
326   // Instantiate the packetizer.
327   R600PacketizerList Packetizer(Fn, ST, MLI);
328 
329   // DFA state table should not be empty.
330   assert(Packetizer.getResourceTracker() && "Empty DFA table!");
331   assert(Packetizer.getResourceTracker()->getInstrItins());
332 
333   if (Packetizer.getResourceTracker()->getInstrItins()->isEmpty())
334     return false;
335 
336   //
337   // Loop over all basic blocks and remove KILL pseudo-instructions
338   // These instructions confuse the dependence analysis. Consider:
339   // D0 = ...   (Insn 0)
340   // R0 = KILL R0, D0 (Insn 1)
341   // R0 = ... (Insn 2)
342   // Here, Insn 1 will result in the dependence graph not emitting an output
343   // dependence between Insn 0 and Insn 2. This can lead to incorrect
344   // packetization
345   //
346   for (MachineBasicBlock &MBB : Fn) {
347     for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
348       if (MI.isKill() || MI.getOpcode() == R600::IMPLICIT_DEF ||
349           (MI.getOpcode() == R600::CF_ALU && !MI.getOperand(8).getImm()))
350         MBB.erase(MI);
351     }
352   }
353 
354   // Loop over all of the basic blocks.
355   for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end();
356        MBB != MBBe; ++MBB) {
357     // Find scheduling regions and schedule / packetize each region.
358     unsigned RemainingCount = MBB->size();
359     for(MachineBasicBlock::iterator RegionEnd = MBB->end();
360         RegionEnd != MBB->begin();) {
361       // The next region starts above the previous region. Look backward in the
362       // instruction stream until we find the nearest boundary.
363       MachineBasicBlock::iterator I = RegionEnd;
364       for(;I != MBB->begin(); --I, --RemainingCount) {
365         if (TII->isSchedulingBoundary(*std::prev(I), &*MBB, Fn))
366           break;
367       }
368       I = MBB->begin();
369 
370       // Skip empty scheduling regions.
371       if (I == RegionEnd) {
372         RegionEnd = std::prev(RegionEnd);
373         --RemainingCount;
374         continue;
375       }
376       // Skip regions with one instruction.
377       if (I == std::prev(RegionEnd)) {
378         RegionEnd = std::prev(RegionEnd);
379         continue;
380       }
381 
382       Packetizer.PacketizeMIs(&*MBB, &*I, RegionEnd);
383       RegionEnd = I;
384     }
385   }
386 
387   return true;
388 
389 }
390 
391 } // end anonymous namespace
392 
393 INITIALIZE_PASS_BEGIN(R600Packetizer, DEBUG_TYPE,
394                      "R600 Packetizer", false, false)
395 INITIALIZE_PASS_END(R600Packetizer, DEBUG_TYPE,
396                     "R600 Packetizer", false, false)
397 
398 char R600Packetizer::ID = 0;
399 
400 char &llvm::R600PacketizerID = R600Packetizer::ID;
401 
402 llvm::FunctionPass *llvm::createR600Packetizer() {
403   return new R600Packetizer();
404 }
405