1 //===- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass compute turns all control flow pseudo instructions into native one 11 /// computing their address on the fly; it also sets STACK_SIZE info. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/R600MCTargetDesc.h" 16 #include "R600.h" 17 #include "R600MachineFunctionInfo.h" 18 #include "R600Subtarget.h" 19 #include "llvm/CodeGen/MachineFunctionPass.h" 20 #include <set> 21 22 using namespace llvm; 23 24 #define DEBUG_TYPE "r600cf" 25 26 namespace { 27 28 struct CFStack { 29 enum StackItem { 30 ENTRY = 0, 31 SUB_ENTRY = 1, 32 FIRST_NON_WQM_PUSH = 2, 33 FIRST_NON_WQM_PUSH_W_FULL_ENTRY = 3 34 }; 35 36 const R600Subtarget *ST; 37 std::vector<StackItem> BranchStack; 38 std::vector<StackItem> LoopStack; 39 unsigned MaxStackSize; 40 unsigned CurrentEntries = 0; 41 unsigned CurrentSubEntries = 0; 42 43 CFStack(const R600Subtarget *st, CallingConv::ID cc) : ST(st), 44 // We need to reserve a stack entry for CALL_FS in vertex shaders. 45 MaxStackSize(cc == CallingConv::AMDGPU_VS ? 1 : 0) {} 46 47 unsigned getLoopDepth(); 48 bool branchStackContains(CFStack::StackItem); 49 bool requiresWorkAroundForInst(unsigned Opcode); 50 unsigned getSubEntrySize(CFStack::StackItem Item); 51 void updateMaxStackSize(); 52 void pushBranch(unsigned Opcode, bool isWQM = false); 53 void pushLoop(); 54 void popBranch(); 55 void popLoop(); 56 }; 57 58 unsigned CFStack::getLoopDepth() { 59 return LoopStack.size(); 60 } 61 62 bool CFStack::branchStackContains(CFStack::StackItem Item) { 63 return llvm::is_contained(BranchStack, Item); 64 } 65 66 bool CFStack::requiresWorkAroundForInst(unsigned Opcode) { 67 if (Opcode == R600::CF_ALU_PUSH_BEFORE && ST->hasCaymanISA() && 68 getLoopDepth() > 1) 69 return true; 70 71 if (!ST->hasCFAluBug()) 72 return false; 73 74 switch(Opcode) { 75 default: return false; 76 case R600::CF_ALU_PUSH_BEFORE: 77 case R600::CF_ALU_ELSE_AFTER: 78 case R600::CF_ALU_BREAK: 79 case R600::CF_ALU_CONTINUE: 80 if (CurrentSubEntries == 0) 81 return false; 82 if (ST->getWavefrontSize() == 64) { 83 // We are being conservative here. We only require this work-around if 84 // CurrentSubEntries > 3 && 85 // (CurrentSubEntries % 4 == 3 || CurrentSubEntries % 4 == 0) 86 // 87 // We have to be conservative, because we don't know for certain that 88 // our stack allocation algorithm for Evergreen/NI is correct. Applying this 89 // work-around when CurrentSubEntries > 3 allows us to over-allocate stack 90 // resources without any problems. 91 return CurrentSubEntries > 3; 92 } else { 93 assert(ST->getWavefrontSize() == 32); 94 // We are being conservative here. We only require the work-around if 95 // CurrentSubEntries > 7 && 96 // (CurrentSubEntries % 8 == 7 || CurrentSubEntries % 8 == 0) 97 // See the comment on the wavefront size == 64 case for why we are 98 // being conservative. 99 return CurrentSubEntries > 7; 100 } 101 } 102 } 103 104 unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) { 105 switch(Item) { 106 default: 107 return 0; 108 case CFStack::FIRST_NON_WQM_PUSH: 109 assert(!ST->hasCaymanISA()); 110 if (ST->getGeneration() <= AMDGPUSubtarget::R700) { 111 // +1 For the push operation. 112 // +2 Extra space required. 113 return 3; 114 } else { 115 // Some documentation says that this is not necessary on Evergreen, 116 // but experimentation has show that we need to allocate 1 extra 117 // sub-entry for the first non-WQM push. 118 // +1 For the push operation. 119 // +1 Extra space required. 120 return 2; 121 } 122 case CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY: 123 assert(ST->getGeneration() >= AMDGPUSubtarget::EVERGREEN); 124 // +1 For the push operation. 125 // +1 Extra space required. 126 return 2; 127 case CFStack::SUB_ENTRY: 128 return 1; 129 } 130 } 131 132 void CFStack::updateMaxStackSize() { 133 unsigned CurrentStackSize = CurrentEntries + divideCeil(CurrentSubEntries, 4); 134 MaxStackSize = std::max(CurrentStackSize, MaxStackSize); 135 } 136 137 void CFStack::pushBranch(unsigned Opcode, bool isWQM) { 138 CFStack::StackItem Item = CFStack::ENTRY; 139 switch(Opcode) { 140 case R600::CF_PUSH_EG: 141 case R600::CF_ALU_PUSH_BEFORE: 142 if (!isWQM) { 143 if (!ST->hasCaymanISA() && 144 !branchStackContains(CFStack::FIRST_NON_WQM_PUSH)) 145 Item = CFStack::FIRST_NON_WQM_PUSH; // May not be required on Evergreen/NI 146 // See comment in 147 // CFStack::getSubEntrySize() 148 else if (CurrentEntries > 0 && 149 ST->getGeneration() > AMDGPUSubtarget::EVERGREEN && 150 !ST->hasCaymanISA() && 151 !branchStackContains(CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY)) 152 Item = CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY; 153 else 154 Item = CFStack::SUB_ENTRY; 155 } else 156 Item = CFStack::ENTRY; 157 break; 158 } 159 BranchStack.push_back(Item); 160 if (Item == CFStack::ENTRY) 161 CurrentEntries++; 162 else 163 CurrentSubEntries += getSubEntrySize(Item); 164 updateMaxStackSize(); 165 } 166 167 void CFStack::pushLoop() { 168 LoopStack.push_back(CFStack::ENTRY); 169 CurrentEntries++; 170 updateMaxStackSize(); 171 } 172 173 void CFStack::popBranch() { 174 CFStack::StackItem Top = BranchStack.back(); 175 if (Top == CFStack::ENTRY) 176 CurrentEntries--; 177 else 178 CurrentSubEntries-= getSubEntrySize(Top); 179 BranchStack.pop_back(); 180 } 181 182 void CFStack::popLoop() { 183 CurrentEntries--; 184 LoopStack.pop_back(); 185 } 186 187 class R600ControlFlowFinalizer : public MachineFunctionPass { 188 private: 189 using ClauseFile = std::pair<MachineInstr *, std::vector<MachineInstr *>>; 190 191 enum ControlFlowInstruction { 192 CF_TC, 193 CF_VC, 194 CF_CALL_FS, 195 CF_WHILE_LOOP, 196 CF_END_LOOP, 197 CF_LOOP_BREAK, 198 CF_LOOP_CONTINUE, 199 CF_JUMP, 200 CF_ELSE, 201 CF_POP, 202 CF_END 203 }; 204 205 const R600InstrInfo *TII = nullptr; 206 const R600RegisterInfo *TRI = nullptr; 207 unsigned MaxFetchInst; 208 const R600Subtarget *ST = nullptr; 209 210 bool IsTrivialInst(MachineInstr &MI) const { 211 switch (MI.getOpcode()) { 212 case R600::KILL: 213 case R600::RETURN: 214 return true; 215 default: 216 return false; 217 } 218 } 219 220 const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const { 221 unsigned Opcode = 0; 222 bool isEg = (ST->getGeneration() >= AMDGPUSubtarget::EVERGREEN); 223 switch (CFI) { 224 case CF_TC: 225 Opcode = isEg ? R600::CF_TC_EG : R600::CF_TC_R600; 226 break; 227 case CF_VC: 228 Opcode = isEg ? R600::CF_VC_EG : R600::CF_VC_R600; 229 break; 230 case CF_CALL_FS: 231 Opcode = isEg ? R600::CF_CALL_FS_EG : R600::CF_CALL_FS_R600; 232 break; 233 case CF_WHILE_LOOP: 234 Opcode = isEg ? R600::WHILE_LOOP_EG : R600::WHILE_LOOP_R600; 235 break; 236 case CF_END_LOOP: 237 Opcode = isEg ? R600::END_LOOP_EG : R600::END_LOOP_R600; 238 break; 239 case CF_LOOP_BREAK: 240 Opcode = isEg ? R600::LOOP_BREAK_EG : R600::LOOP_BREAK_R600; 241 break; 242 case CF_LOOP_CONTINUE: 243 Opcode = isEg ? R600::CF_CONTINUE_EG : R600::CF_CONTINUE_R600; 244 break; 245 case CF_JUMP: 246 Opcode = isEg ? R600::CF_JUMP_EG : R600::CF_JUMP_R600; 247 break; 248 case CF_ELSE: 249 Opcode = isEg ? R600::CF_ELSE_EG : R600::CF_ELSE_R600; 250 break; 251 case CF_POP: 252 Opcode = isEg ? R600::POP_EG : R600::POP_R600; 253 break; 254 case CF_END: 255 if (ST->hasCaymanISA()) { 256 Opcode = R600::CF_END_CM; 257 break; 258 } 259 Opcode = isEg ? R600::CF_END_EG : R600::CF_END_R600; 260 break; 261 } 262 assert (Opcode && "No opcode selected"); 263 return TII->get(Opcode); 264 } 265 266 bool isCompatibleWithClause(const MachineInstr &MI, 267 std::set<unsigned> &DstRegs) const { 268 unsigned DstMI, SrcMI; 269 for (MachineInstr::const_mop_iterator I = MI.operands_begin(), 270 E = MI.operands_end(); 271 I != E; ++I) { 272 const MachineOperand &MO = *I; 273 if (!MO.isReg()) 274 continue; 275 if (MO.isDef()) { 276 Register Reg = MO.getReg(); 277 if (R600::R600_Reg128RegClass.contains(Reg)) 278 DstMI = Reg; 279 else 280 DstMI = TRI->getMatchingSuperReg(Reg, 281 R600RegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)), 282 &R600::R600_Reg128RegClass); 283 } 284 if (MO.isUse()) { 285 Register Reg = MO.getReg(); 286 if (R600::R600_Reg128RegClass.contains(Reg)) 287 SrcMI = Reg; 288 else 289 SrcMI = TRI->getMatchingSuperReg(Reg, 290 R600RegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)), 291 &R600::R600_Reg128RegClass); 292 } 293 } 294 if ((DstRegs.find(SrcMI) == DstRegs.end())) { 295 DstRegs.insert(DstMI); 296 return true; 297 } else 298 return false; 299 } 300 301 ClauseFile 302 MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I) 303 const { 304 MachineBasicBlock::iterator ClauseHead = I; 305 std::vector<MachineInstr *> ClauseContent; 306 unsigned AluInstCount = 0; 307 bool IsTex = TII->usesTextureCache(*ClauseHead); 308 std::set<unsigned> DstRegs; 309 for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) { 310 if (IsTrivialInst(*I)) 311 continue; 312 if (AluInstCount >= MaxFetchInst) 313 break; 314 if ((IsTex && !TII->usesTextureCache(*I)) || 315 (!IsTex && !TII->usesVertexCache(*I))) 316 break; 317 if (!isCompatibleWithClause(*I, DstRegs)) 318 break; 319 AluInstCount ++; 320 ClauseContent.push_back(&*I); 321 } 322 MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead), 323 getHWInstrDesc(IsTex?CF_TC:CF_VC)) 324 .addImm(0) // ADDR 325 .addImm(AluInstCount - 1); // COUNT 326 return ClauseFile(MIb, std::move(ClauseContent)); 327 } 328 329 void getLiteral(MachineInstr &MI, std::vector<MachineOperand *> &Lits) const { 330 static const unsigned LiteralRegs[] = { 331 R600::ALU_LITERAL_X, 332 R600::ALU_LITERAL_Y, 333 R600::ALU_LITERAL_Z, 334 R600::ALU_LITERAL_W 335 }; 336 const SmallVector<std::pair<MachineOperand *, int64_t>, 3> Srcs = 337 TII->getSrcs(MI); 338 for (const auto &Src:Srcs) { 339 if (Src.first->getReg() != R600::ALU_LITERAL_X) 340 continue; 341 int64_t Imm = Src.second; 342 std::vector<MachineOperand *>::iterator It = 343 llvm::find_if(Lits, [&](MachineOperand *val) { 344 return val->isImm() && (val->getImm() == Imm); 345 }); 346 347 // Get corresponding Operand 348 MachineOperand &Operand = MI.getOperand( 349 TII->getOperandIdx(MI.getOpcode(), R600::OpName::literal)); 350 351 if (It != Lits.end()) { 352 // Reuse existing literal reg 353 unsigned Index = It - Lits.begin(); 354 Src.first->setReg(LiteralRegs[Index]); 355 } else { 356 // Allocate new literal reg 357 assert(Lits.size() < 4 && "Too many literals in Instruction Group"); 358 Src.first->setReg(LiteralRegs[Lits.size()]); 359 Lits.push_back(&Operand); 360 } 361 } 362 } 363 364 MachineBasicBlock::iterator insertLiterals( 365 MachineBasicBlock::iterator InsertPos, 366 const std::vector<unsigned> &Literals) const { 367 MachineBasicBlock *MBB = InsertPos->getParent(); 368 for (unsigned i = 0, e = Literals.size(); i < e; i+=2) { 369 unsigned LiteralPair0 = Literals[i]; 370 unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0; 371 InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(), 372 TII->get(R600::LITERALS)) 373 .addImm(LiteralPair0) 374 .addImm(LiteralPair1); 375 } 376 return InsertPos; 377 } 378 379 ClauseFile 380 MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I) 381 const { 382 MachineInstr &ClauseHead = *I; 383 std::vector<MachineInstr *> ClauseContent; 384 I++; 385 for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) { 386 if (IsTrivialInst(*I)) { 387 ++I; 388 continue; 389 } 390 if (!I->isBundle() && !TII->isALUInstr(I->getOpcode())) 391 break; 392 std::vector<MachineOperand *>Literals; 393 if (I->isBundle()) { 394 MachineInstr &DeleteMI = *I; 395 MachineBasicBlock::instr_iterator BI = I.getInstrIterator(); 396 while (++BI != E && BI->isBundledWithPred()) { 397 BI->unbundleFromPred(); 398 for (MachineOperand &MO : BI->operands()) { 399 if (MO.isReg() && MO.isInternalRead()) 400 MO.setIsInternalRead(false); 401 } 402 getLiteral(*BI, Literals); 403 ClauseContent.push_back(&*BI); 404 } 405 I = BI; 406 DeleteMI.eraseFromParent(); 407 } else { 408 getLiteral(*I, Literals); 409 ClauseContent.push_back(&*I); 410 I++; 411 } 412 for (unsigned i = 0, e = Literals.size(); i < e; i += 2) { 413 MachineInstrBuilder MILit = BuildMI(MBB, I, I->getDebugLoc(), 414 TII->get(R600::LITERALS)); 415 if (Literals[i]->isImm()) { 416 MILit.addImm(Literals[i]->getImm()); 417 } else { 418 MILit.addGlobalAddress(Literals[i]->getGlobal(), 419 Literals[i]->getOffset()); 420 } 421 if (i + 1 < e) { 422 if (Literals[i + 1]->isImm()) { 423 MILit.addImm(Literals[i + 1]->getImm()); 424 } else { 425 MILit.addGlobalAddress(Literals[i + 1]->getGlobal(), 426 Literals[i + 1]->getOffset()); 427 } 428 } else 429 MILit.addImm(0); 430 ClauseContent.push_back(MILit); 431 } 432 } 433 assert(ClauseContent.size() < 128 && "ALU clause is too big"); 434 ClauseHead.getOperand(7).setImm(ClauseContent.size() - 1); 435 return ClauseFile(&ClauseHead, std::move(ClauseContent)); 436 } 437 438 void EmitFetchClause(MachineBasicBlock::iterator InsertPos, 439 const DebugLoc &DL, ClauseFile &Clause, 440 unsigned &CfCount) { 441 CounterPropagateAddr(*Clause.first, CfCount); 442 MachineBasicBlock *BB = Clause.first->getParent(); 443 BuildMI(BB, DL, TII->get(R600::FETCH_CLAUSE)).addImm(CfCount); 444 for (MachineInstr *MI : Clause.second) 445 BB->splice(InsertPos, BB, MI); 446 CfCount += 2 * Clause.second.size(); 447 } 448 449 void EmitALUClause(MachineBasicBlock::iterator InsertPos, const DebugLoc &DL, 450 ClauseFile &Clause, unsigned &CfCount) { 451 Clause.first->getOperand(0).setImm(0); 452 CounterPropagateAddr(*Clause.first, CfCount); 453 MachineBasicBlock *BB = Clause.first->getParent(); 454 BuildMI(BB, DL, TII->get(R600::ALU_CLAUSE)).addImm(CfCount); 455 for (MachineInstr *MI : Clause.second) 456 BB->splice(InsertPos, BB, MI); 457 CfCount += Clause.second.size(); 458 } 459 460 void CounterPropagateAddr(MachineInstr &MI, unsigned Addr) const { 461 MI.getOperand(0).setImm(Addr + MI.getOperand(0).getImm()); 462 } 463 void CounterPropagateAddr(const std::set<MachineInstr *> &MIs, 464 unsigned Addr) const { 465 for (MachineInstr *MI : MIs) { 466 CounterPropagateAddr(*MI, Addr); 467 } 468 } 469 470 public: 471 static char ID; 472 473 R600ControlFlowFinalizer() : MachineFunctionPass(ID) {} 474 475 bool runOnMachineFunction(MachineFunction &MF) override { 476 ST = &MF.getSubtarget<R600Subtarget>(); 477 MaxFetchInst = ST->getTexVTXClauseSize(); 478 TII = ST->getInstrInfo(); 479 TRI = ST->getRegisterInfo(); 480 481 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 482 483 CFStack CFStack(ST, MF.getFunction().getCallingConv()); 484 for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME; 485 ++MB) { 486 MachineBasicBlock &MBB = *MB; 487 unsigned CfCount = 0; 488 std::vector<std::pair<unsigned, std::set<MachineInstr *>>> LoopStack; 489 std::vector<MachineInstr * > IfThenElseStack; 490 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_VS) { 491 BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), 492 getHWInstrDesc(CF_CALL_FS)); 493 CfCount++; 494 } 495 std::vector<ClauseFile> FetchClauses, AluClauses; 496 std::vector<MachineInstr *> LastAlu(1); 497 std::vector<MachineInstr *> ToPopAfter; 498 499 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); 500 I != E;) { 501 if (TII->usesTextureCache(*I) || TII->usesVertexCache(*I)) { 502 LLVM_DEBUG(dbgs() << CfCount << ":"; I->dump();); 503 FetchClauses.push_back(MakeFetchClause(MBB, I)); 504 CfCount++; 505 LastAlu.back() = nullptr; 506 continue; 507 } 508 509 MachineBasicBlock::iterator MI = I; 510 if (MI->getOpcode() != R600::ENDIF) 511 LastAlu.back() = nullptr; 512 if (MI->getOpcode() == R600::CF_ALU) 513 LastAlu.back() = &*MI; 514 I++; 515 bool RequiresWorkAround = 516 CFStack.requiresWorkAroundForInst(MI->getOpcode()); 517 switch (MI->getOpcode()) { 518 case R600::CF_ALU_PUSH_BEFORE: 519 if (RequiresWorkAround) { 520 LLVM_DEBUG(dbgs() 521 << "Applying bug work-around for ALU_PUSH_BEFORE\n"); 522 BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(R600::CF_PUSH_EG)) 523 .addImm(CfCount + 1) 524 .addImm(1); 525 MI->setDesc(TII->get(R600::CF_ALU)); 526 CfCount++; 527 CFStack.pushBranch(R600::CF_PUSH_EG); 528 } else 529 CFStack.pushBranch(R600::CF_ALU_PUSH_BEFORE); 530 [[fallthrough]]; 531 case R600::CF_ALU: 532 I = MI; 533 AluClauses.push_back(MakeALUClause(MBB, I)); 534 LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump();); 535 CfCount++; 536 break; 537 case R600::WHILELOOP: { 538 CFStack.pushLoop(); 539 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 540 getHWInstrDesc(CF_WHILE_LOOP)) 541 .addImm(1); 542 std::pair<unsigned, std::set<MachineInstr *>> Pair(CfCount, 543 std::set<MachineInstr *>()); 544 Pair.second.insert(MIb); 545 LoopStack.push_back(std::move(Pair)); 546 MI->eraseFromParent(); 547 CfCount++; 548 break; 549 } 550 case R600::ENDLOOP: { 551 CFStack.popLoop(); 552 std::pair<unsigned, std::set<MachineInstr *>> Pair = 553 std::move(LoopStack.back()); 554 LoopStack.pop_back(); 555 CounterPropagateAddr(Pair.second, CfCount); 556 BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP)) 557 .addImm(Pair.first + 1); 558 MI->eraseFromParent(); 559 CfCount++; 560 break; 561 } 562 case R600::IF_PREDICATE_SET: { 563 LastAlu.push_back(nullptr); 564 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 565 getHWInstrDesc(CF_JUMP)) 566 .addImm(0) 567 .addImm(0); 568 IfThenElseStack.push_back(MIb); 569 LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump();); 570 MI->eraseFromParent(); 571 CfCount++; 572 break; 573 } 574 case R600::ELSE: { 575 MachineInstr * JumpInst = IfThenElseStack.back(); 576 IfThenElseStack.pop_back(); 577 CounterPropagateAddr(*JumpInst, CfCount); 578 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 579 getHWInstrDesc(CF_ELSE)) 580 .addImm(0) 581 .addImm(0); 582 LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump();); 583 IfThenElseStack.push_back(MIb); 584 MI->eraseFromParent(); 585 CfCount++; 586 break; 587 } 588 case R600::ENDIF: { 589 CFStack.popBranch(); 590 if (LastAlu.back()) { 591 ToPopAfter.push_back(LastAlu.back()); 592 } else { 593 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 594 getHWInstrDesc(CF_POP)) 595 .addImm(CfCount + 1) 596 .addImm(1); 597 (void)MIb; 598 LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump();); 599 CfCount++; 600 } 601 602 MachineInstr *IfOrElseInst = IfThenElseStack.back(); 603 IfThenElseStack.pop_back(); 604 CounterPropagateAddr(*IfOrElseInst, CfCount); 605 IfOrElseInst->getOperand(1).setImm(1); 606 LastAlu.pop_back(); 607 MI->eraseFromParent(); 608 break; 609 } 610 case R600::BREAK: { 611 CfCount ++; 612 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 613 getHWInstrDesc(CF_LOOP_BREAK)) 614 .addImm(0); 615 LoopStack.back().second.insert(MIb); 616 MI->eraseFromParent(); 617 break; 618 } 619 case R600::CONTINUE: { 620 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 621 getHWInstrDesc(CF_LOOP_CONTINUE)) 622 .addImm(0); 623 LoopStack.back().second.insert(MIb); 624 MI->eraseFromParent(); 625 CfCount++; 626 break; 627 } 628 case R600::RETURN: { 629 DebugLoc DL = MBB.findDebugLoc(MI); 630 BuildMI(MBB, MI, DL, getHWInstrDesc(CF_END)); 631 CfCount++; 632 if (CfCount % 2) { 633 BuildMI(MBB, I, DL, TII->get(R600::PAD)); 634 CfCount++; 635 } 636 MI->eraseFromParent(); 637 for (ClauseFile &CF : FetchClauses) 638 EmitFetchClause(I, DL, CF, CfCount); 639 for (ClauseFile &CF : AluClauses) 640 EmitALUClause(I, DL, CF, CfCount); 641 break; 642 } 643 default: 644 if (TII->isExport(MI->getOpcode())) { 645 LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump();); 646 CfCount++; 647 } 648 break; 649 } 650 } 651 for (MachineInstr *Alu : ToPopAfter) { 652 BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu), 653 TII->get(R600::CF_ALU_POP_AFTER)) 654 .addImm(Alu->getOperand(0).getImm()) 655 .addImm(Alu->getOperand(1).getImm()) 656 .addImm(Alu->getOperand(2).getImm()) 657 .addImm(Alu->getOperand(3).getImm()) 658 .addImm(Alu->getOperand(4).getImm()) 659 .addImm(Alu->getOperand(5).getImm()) 660 .addImm(Alu->getOperand(6).getImm()) 661 .addImm(Alu->getOperand(7).getImm()) 662 .addImm(Alu->getOperand(8).getImm()); 663 Alu->eraseFromParent(); 664 } 665 MFI->CFStackSize = CFStack.MaxStackSize; 666 } 667 668 return false; 669 } 670 671 StringRef getPassName() const override { 672 return "R600 Control Flow Finalizer Pass"; 673 } 674 }; 675 676 } // end anonymous namespace 677 678 INITIALIZE_PASS_BEGIN(R600ControlFlowFinalizer, DEBUG_TYPE, 679 "R600 Control Flow Finalizer", false, false) 680 INITIALIZE_PASS_END(R600ControlFlowFinalizer, DEBUG_TYPE, 681 "R600 Control Flow Finalizer", false, false) 682 683 char R600ControlFlowFinalizer::ID = 0; 684 685 char &llvm::R600ControlFlowFinalizerID = R600ControlFlowFinalizer::ID; 686 687 FunctionPass *llvm::createR600ControlFlowFinalizer() { 688 return new R600ControlFlowFinalizer(); 689 } 690