1 //===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass implements instructions packetization for R600. It unsets isLast 11 /// bit of instructions inside a bundle and substitutes src register with 12 /// PreviousVector when applicable. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #include "MCTargetDesc/R600MCTargetDesc.h" 17 #include "R600.h" 18 #include "R600Subtarget.h" 19 #include "llvm/CodeGen/DFAPacketizer.h" 20 #include "llvm/CodeGen/MachineDominators.h" 21 #include "llvm/CodeGen/MachineLoopInfo.h" 22 #include "llvm/CodeGen/ScheduleDAG.h" 23 24 using namespace llvm; 25 26 #define DEBUG_TYPE "packets" 27 28 namespace { 29 30 class R600Packetizer : public MachineFunctionPass { 31 32 public: 33 static char ID; 34 R600Packetizer() : MachineFunctionPass(ID) {} 35 36 void getAnalysisUsage(AnalysisUsage &AU) const override { 37 AU.setPreservesCFG(); 38 AU.addRequired<MachineDominatorTree>(); 39 AU.addPreserved<MachineDominatorTree>(); 40 AU.addRequired<MachineLoopInfo>(); 41 AU.addPreserved<MachineLoopInfo>(); 42 MachineFunctionPass::getAnalysisUsage(AU); 43 } 44 45 StringRef getPassName() const override { return "R600 Packetizer"; } 46 47 bool runOnMachineFunction(MachineFunction &Fn) override; 48 }; 49 50 class R600PacketizerList : public VLIWPacketizerList { 51 private: 52 const R600InstrInfo *TII; 53 const R600RegisterInfo &TRI; 54 bool VLIW5; 55 bool ConsideredInstUsesAlreadyWrittenVectorElement; 56 57 unsigned getSlot(const MachineInstr &MI) const { 58 return TRI.getHWRegChan(MI.getOperand(0).getReg()); 59 } 60 61 /// \returns register to PV chan mapping for bundle/single instructions that 62 /// immediately precedes I. 63 DenseMap<unsigned, unsigned> getPreviousVector(MachineBasicBlock::iterator I) 64 const { 65 DenseMap<unsigned, unsigned> Result; 66 I--; 67 if (!TII->isALUInstr(I->getOpcode()) && !I->isBundle()) 68 return Result; 69 MachineBasicBlock::instr_iterator BI = I.getInstrIterator(); 70 if (I->isBundle()) 71 BI++; 72 int LastDstChan = -1; 73 do { 74 bool isTrans = false; 75 int BISlot = getSlot(*BI); 76 if (LastDstChan >= BISlot) 77 isTrans = true; 78 LastDstChan = BISlot; 79 if (TII->isPredicated(*BI)) 80 continue; 81 int OperandIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::write); 82 if (OperandIdx > -1 && BI->getOperand(OperandIdx).getImm() == 0) 83 continue; 84 int DstIdx = TII->getOperandIdx(BI->getOpcode(), R600::OpName::dst); 85 if (DstIdx == -1) { 86 continue; 87 } 88 Register Dst = BI->getOperand(DstIdx).getReg(); 89 if (isTrans || TII->isTransOnly(*BI)) { 90 Result[Dst] = R600::PS; 91 continue; 92 } 93 if (BI->getOpcode() == R600::DOT4_r600 || 94 BI->getOpcode() == R600::DOT4_eg) { 95 Result[Dst] = R600::PV_X; 96 continue; 97 } 98 if (Dst == R600::OQAP) { 99 continue; 100 } 101 unsigned PVReg = 0; 102 switch (TRI.getHWRegChan(Dst)) { 103 case 0: 104 PVReg = R600::PV_X; 105 break; 106 case 1: 107 PVReg = R600::PV_Y; 108 break; 109 case 2: 110 PVReg = R600::PV_Z; 111 break; 112 case 3: 113 PVReg = R600::PV_W; 114 break; 115 default: 116 llvm_unreachable("Invalid Chan"); 117 } 118 Result[Dst] = PVReg; 119 } while ((++BI)->isBundledWithPred()); 120 return Result; 121 } 122 123 void substitutePV(MachineInstr &MI, const DenseMap<unsigned, unsigned> &PVs) 124 const { 125 unsigned Ops[] = { 126 R600::OpName::src0, 127 R600::OpName::src1, 128 R600::OpName::src2 129 }; 130 for (unsigned Op : Ops) { 131 int OperandIdx = TII->getOperandIdx(MI.getOpcode(), Op); 132 if (OperandIdx < 0) 133 continue; 134 Register Src = MI.getOperand(OperandIdx).getReg(); 135 const DenseMap<unsigned, unsigned>::const_iterator It = PVs.find(Src); 136 if (It != PVs.end()) 137 MI.getOperand(OperandIdx).setReg(It->second); 138 } 139 } 140 public: 141 // Ctor. 142 R600PacketizerList(MachineFunction &MF, const R600Subtarget &ST, 143 MachineLoopInfo &MLI) 144 : VLIWPacketizerList(MF, MLI, nullptr), 145 TII(ST.getInstrInfo()), 146 TRI(TII->getRegisterInfo()) { 147 VLIW5 = !ST.hasCaymanISA(); 148 } 149 150 // initPacketizerState - initialize some internal flags. 151 void initPacketizerState() override { 152 ConsideredInstUsesAlreadyWrittenVectorElement = false; 153 } 154 155 // ignorePseudoInstruction - Ignore bundling of pseudo instructions. 156 bool ignorePseudoInstruction(const MachineInstr &MI, 157 const MachineBasicBlock *MBB) override { 158 return false; 159 } 160 161 // isSoloInstruction - return true if instruction MI can not be packetized 162 // with any other instruction, which means that MI itself is a packet. 163 bool isSoloInstruction(const MachineInstr &MI) override { 164 if (TII->isVector(MI)) 165 return true; 166 if (!TII->isALUInstr(MI.getOpcode())) 167 return true; 168 if (MI.getOpcode() == R600::GROUP_BARRIER) 169 return true; 170 // XXX: This can be removed once the packetizer properly handles all the 171 // LDS instruction group restrictions. 172 return TII->isLDSInstr(MI.getOpcode()); 173 } 174 175 // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ 176 // together. 177 bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override { 178 MachineInstr *MII = SUI->getInstr(), *MIJ = SUJ->getInstr(); 179 if (getSlot(*MII) == getSlot(*MIJ)) 180 ConsideredInstUsesAlreadyWrittenVectorElement = true; 181 // Does MII and MIJ share the same pred_sel ? 182 int OpI = TII->getOperandIdx(MII->getOpcode(), R600::OpName::pred_sel), 183 OpJ = TII->getOperandIdx(MIJ->getOpcode(), R600::OpName::pred_sel); 184 Register PredI = (OpI > -1)?MII->getOperand(OpI).getReg() : Register(), 185 PredJ = (OpJ > -1)?MIJ->getOperand(OpJ).getReg() : Register(); 186 if (PredI != PredJ) 187 return false; 188 if (SUJ->isSucc(SUI)) { 189 for (unsigned i = 0, e = SUJ->Succs.size(); i < e; ++i) { 190 const SDep &Dep = SUJ->Succs[i]; 191 if (Dep.getSUnit() != SUI) 192 continue; 193 if (Dep.getKind() == SDep::Anti) 194 continue; 195 if (Dep.getKind() == SDep::Output) 196 if (MII->getOperand(0).getReg() != MIJ->getOperand(0).getReg()) 197 continue; 198 return false; 199 } 200 } 201 202 bool ARDef = 203 TII->definesAddressRegister(*MII) || TII->definesAddressRegister(*MIJ); 204 bool ARUse = 205 TII->usesAddressRegister(*MII) || TII->usesAddressRegister(*MIJ); 206 207 return !ARDef || !ARUse; 208 } 209 210 // isLegalToPruneDependencies - Is it legal to prune dependency between SUI 211 // and SUJ. 212 bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override { 213 return false; 214 } 215 216 void setIsLastBit(MachineInstr *MI, unsigned Bit) const { 217 unsigned LastOp = TII->getOperandIdx(MI->getOpcode(), R600::OpName::last); 218 MI->getOperand(LastOp).setImm(Bit); 219 } 220 221 bool isBundlableWithCurrentPMI(MachineInstr &MI, 222 const DenseMap<unsigned, unsigned> &PV, 223 std::vector<R600InstrInfo::BankSwizzle> &BS, 224 bool &isTransSlot) { 225 isTransSlot = TII->isTransOnly(MI); 226 assert (!isTransSlot || VLIW5); 227 228 // Is the dst reg sequence legal ? 229 if (!isTransSlot && !CurrentPacketMIs.empty()) { 230 if (getSlot(MI) <= getSlot(*CurrentPacketMIs.back())) { 231 if (ConsideredInstUsesAlreadyWrittenVectorElement && 232 !TII->isVectorOnly(MI) && VLIW5) { 233 isTransSlot = true; 234 LLVM_DEBUG({ 235 dbgs() << "Considering as Trans Inst :"; 236 MI.dump(); 237 }); 238 } 239 else 240 return false; 241 } 242 } 243 244 // Are the Constants limitations met ? 245 CurrentPacketMIs.push_back(&MI); 246 if (!TII->fitsConstReadLimitations(CurrentPacketMIs)) { 247 LLVM_DEBUG({ 248 dbgs() << "Couldn't pack :\n"; 249 MI.dump(); 250 dbgs() << "with the following packets :\n"; 251 for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) { 252 CurrentPacketMIs[i]->dump(); 253 dbgs() << "\n"; 254 } 255 dbgs() << "because of Consts read limitations\n"; 256 }); 257 CurrentPacketMIs.pop_back(); 258 return false; 259 } 260 261 // Is there a BankSwizzle set that meet Read Port limitations ? 262 if (!TII->fitsReadPortLimitations(CurrentPacketMIs, 263 PV, BS, isTransSlot)) { 264 LLVM_DEBUG({ 265 dbgs() << "Couldn't pack :\n"; 266 MI.dump(); 267 dbgs() << "with the following packets :\n"; 268 for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) { 269 CurrentPacketMIs[i]->dump(); 270 dbgs() << "\n"; 271 } 272 dbgs() << "because of Read port limitations\n"; 273 }); 274 CurrentPacketMIs.pop_back(); 275 return false; 276 } 277 278 // We cannot read LDS source registers from the Trans slot. 279 if (isTransSlot && TII->readsLDSSrcReg(MI)) 280 return false; 281 282 CurrentPacketMIs.pop_back(); 283 return true; 284 } 285 286 MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override { 287 MachineBasicBlock::iterator FirstInBundle = 288 CurrentPacketMIs.empty() ? &MI : CurrentPacketMIs.front(); 289 const DenseMap<unsigned, unsigned> &PV = 290 getPreviousVector(FirstInBundle); 291 std::vector<R600InstrInfo::BankSwizzle> BS; 292 bool isTransSlot; 293 294 if (isBundlableWithCurrentPMI(MI, PV, BS, isTransSlot)) { 295 for (unsigned i = 0, e = CurrentPacketMIs.size(); i < e; i++) { 296 MachineInstr *MI = CurrentPacketMIs[i]; 297 unsigned Op = TII->getOperandIdx(MI->getOpcode(), 298 R600::OpName::bank_swizzle); 299 MI->getOperand(Op).setImm(BS[i]); 300 } 301 unsigned Op = 302 TII->getOperandIdx(MI.getOpcode(), R600::OpName::bank_swizzle); 303 MI.getOperand(Op).setImm(BS.back()); 304 if (!CurrentPacketMIs.empty()) 305 setIsLastBit(CurrentPacketMIs.back(), 0); 306 substitutePV(MI, PV); 307 MachineBasicBlock::iterator It = VLIWPacketizerList::addToPacket(MI); 308 if (isTransSlot) { 309 endPacket(std::next(It)->getParent(), std::next(It)); 310 } 311 return It; 312 } 313 endPacket(MI.getParent(), MI); 314 if (TII->isTransOnly(MI)) 315 return MI; 316 return VLIWPacketizerList::addToPacket(MI); 317 } 318 }; 319 320 bool R600Packetizer::runOnMachineFunction(MachineFunction &Fn) { 321 const R600Subtarget &ST = Fn.getSubtarget<R600Subtarget>(); 322 const R600InstrInfo *TII = ST.getInstrInfo(); 323 324 MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>(); 325 326 // Instantiate the packetizer. 327 R600PacketizerList Packetizer(Fn, ST, MLI); 328 329 // DFA state table should not be empty. 330 assert(Packetizer.getResourceTracker() && "Empty DFA table!"); 331 assert(Packetizer.getResourceTracker()->getInstrItins()); 332 333 if (Packetizer.getResourceTracker()->getInstrItins()->isEmpty()) 334 return false; 335 336 // 337 // Loop over all basic blocks and remove KILL pseudo-instructions 338 // These instructions confuse the dependence analysis. Consider: 339 // D0 = ... (Insn 0) 340 // R0 = KILL R0, D0 (Insn 1) 341 // R0 = ... (Insn 2) 342 // Here, Insn 1 will result in the dependence graph not emitting an output 343 // dependence between Insn 0 and Insn 2. This can lead to incorrect 344 // packetization 345 // 346 for (MachineBasicBlock &MBB : Fn) { 347 for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) { 348 if (MI.isKill() || MI.getOpcode() == R600::IMPLICIT_DEF || 349 (MI.getOpcode() == R600::CF_ALU && !MI.getOperand(8).getImm())) 350 MBB.erase(MI); 351 } 352 } 353 354 // Loop over all of the basic blocks. 355 for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); 356 MBB != MBBe; ++MBB) { 357 // Find scheduling regions and schedule / packetize each region. 358 unsigned RemainingCount = MBB->size(); 359 for(MachineBasicBlock::iterator RegionEnd = MBB->end(); 360 RegionEnd != MBB->begin();) { 361 // The next region starts above the previous region. Look backward in the 362 // instruction stream until we find the nearest boundary. 363 MachineBasicBlock::iterator I = RegionEnd; 364 for(;I != MBB->begin(); --I, --RemainingCount) { 365 if (TII->isSchedulingBoundary(*std::prev(I), &*MBB, Fn)) 366 break; 367 } 368 I = MBB->begin(); 369 370 // Skip empty scheduling regions. 371 if (I == RegionEnd) { 372 RegionEnd = std::prev(RegionEnd); 373 --RemainingCount; 374 continue; 375 } 376 // Skip regions with one instruction. 377 if (I == std::prev(RegionEnd)) { 378 RegionEnd = std::prev(RegionEnd); 379 continue; 380 } 381 382 Packetizer.PacketizeMIs(&*MBB, &*I, RegionEnd); 383 RegionEnd = I; 384 } 385 } 386 387 return true; 388 389 } 390 391 } // end anonymous namespace 392 393 INITIALIZE_PASS_BEGIN(R600Packetizer, DEBUG_TYPE, 394 "R600 Packetizer", false, false) 395 INITIALIZE_PASS_END(R600Packetizer, DEBUG_TYPE, 396 "R600 Packetizer", false, false) 397 398 char R600Packetizer::ID = 0; 399 400 char &llvm::R600PacketizerID = R600Packetizer::ID; 401 402 llvm::FunctionPass *llvm::createR600Packetizer() { 403 return new R600Packetizer(); 404 } 405