1 //===- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This pass compute turns all control flow pseudo instructions into native one 11 /// computing their address on the fly; it also sets STACK_SIZE info. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "MCTargetDesc/R600MCTargetDesc.h" 16 #include "R600.h" 17 #include "R600MachineFunctionInfo.h" 18 #include "R600Subtarget.h" 19 #include <set> 20 21 using namespace llvm; 22 23 #define DEBUG_TYPE "r600cf" 24 25 namespace { 26 27 struct CFStack { 28 enum StackItem { 29 ENTRY = 0, 30 SUB_ENTRY = 1, 31 FIRST_NON_WQM_PUSH = 2, 32 FIRST_NON_WQM_PUSH_W_FULL_ENTRY = 3 33 }; 34 35 const R600Subtarget *ST; 36 std::vector<StackItem> BranchStack; 37 std::vector<StackItem> LoopStack; 38 unsigned MaxStackSize; 39 unsigned CurrentEntries = 0; 40 unsigned CurrentSubEntries = 0; 41 42 CFStack(const R600Subtarget *st, CallingConv::ID cc) : ST(st), 43 // We need to reserve a stack entry for CALL_FS in vertex shaders. 44 MaxStackSize(cc == CallingConv::AMDGPU_VS ? 1 : 0) {} 45 46 unsigned getLoopDepth(); 47 bool branchStackContains(CFStack::StackItem); 48 bool requiresWorkAroundForInst(unsigned Opcode); 49 unsigned getSubEntrySize(CFStack::StackItem Item); 50 void updateMaxStackSize(); 51 void pushBranch(unsigned Opcode, bool isWQM = false); 52 void pushLoop(); 53 void popBranch(); 54 void popLoop(); 55 }; 56 57 unsigned CFStack::getLoopDepth() { 58 return LoopStack.size(); 59 } 60 61 bool CFStack::branchStackContains(CFStack::StackItem Item) { 62 return llvm::is_contained(BranchStack, Item); 63 } 64 65 bool CFStack::requiresWorkAroundForInst(unsigned Opcode) { 66 if (Opcode == R600::CF_ALU_PUSH_BEFORE && ST->hasCaymanISA() && 67 getLoopDepth() > 1) 68 return true; 69 70 if (!ST->hasCFAluBug()) 71 return false; 72 73 switch(Opcode) { 74 default: return false; 75 case R600::CF_ALU_PUSH_BEFORE: 76 case R600::CF_ALU_ELSE_AFTER: 77 case R600::CF_ALU_BREAK: 78 case R600::CF_ALU_CONTINUE: 79 if (CurrentSubEntries == 0) 80 return false; 81 if (ST->getWavefrontSize() == 64) { 82 // We are being conservative here. We only require this work-around if 83 // CurrentSubEntries > 3 && 84 // (CurrentSubEntries % 4 == 3 || CurrentSubEntries % 4 == 0) 85 // 86 // We have to be conservative, because we don't know for certain that 87 // our stack allocation algorithm for Evergreen/NI is correct. Applying this 88 // work-around when CurrentSubEntries > 3 allows us to over-allocate stack 89 // resources without any problems. 90 return CurrentSubEntries > 3; 91 } else { 92 assert(ST->getWavefrontSize() == 32); 93 // We are being conservative here. We only require the work-around if 94 // CurrentSubEntries > 7 && 95 // (CurrentSubEntries % 8 == 7 || CurrentSubEntries % 8 == 0) 96 // See the comment on the wavefront size == 64 case for why we are 97 // being conservative. 98 return CurrentSubEntries > 7; 99 } 100 } 101 } 102 103 unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) { 104 switch(Item) { 105 default: 106 return 0; 107 case CFStack::FIRST_NON_WQM_PUSH: 108 assert(!ST->hasCaymanISA()); 109 if (ST->getGeneration() <= AMDGPUSubtarget::R700) { 110 // +1 For the push operation. 111 // +2 Extra space required. 112 return 3; 113 } else { 114 // Some documentation says that this is not necessary on Evergreen, 115 // but experimentation has show that we need to allocate 1 extra 116 // sub-entry for the first non-WQM push. 117 // +1 For the push operation. 118 // +1 Extra space required. 119 return 2; 120 } 121 case CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY: 122 assert(ST->getGeneration() >= AMDGPUSubtarget::EVERGREEN); 123 // +1 For the push operation. 124 // +1 Extra space required. 125 return 2; 126 case CFStack::SUB_ENTRY: 127 return 1; 128 } 129 } 130 131 void CFStack::updateMaxStackSize() { 132 unsigned CurrentStackSize = CurrentEntries + divideCeil(CurrentSubEntries, 4); 133 MaxStackSize = std::max(CurrentStackSize, MaxStackSize); 134 } 135 136 void CFStack::pushBranch(unsigned Opcode, bool isWQM) { 137 CFStack::StackItem Item = CFStack::ENTRY; 138 switch(Opcode) { 139 case R600::CF_PUSH_EG: 140 case R600::CF_ALU_PUSH_BEFORE: 141 if (!isWQM) { 142 if (!ST->hasCaymanISA() && 143 !branchStackContains(CFStack::FIRST_NON_WQM_PUSH)) 144 Item = CFStack::FIRST_NON_WQM_PUSH; // May not be required on Evergreen/NI 145 // See comment in 146 // CFStack::getSubEntrySize() 147 else if (CurrentEntries > 0 && 148 ST->getGeneration() > AMDGPUSubtarget::EVERGREEN && 149 !ST->hasCaymanISA() && 150 !branchStackContains(CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY)) 151 Item = CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY; 152 else 153 Item = CFStack::SUB_ENTRY; 154 } else 155 Item = CFStack::ENTRY; 156 break; 157 } 158 BranchStack.push_back(Item); 159 if (Item == CFStack::ENTRY) 160 CurrentEntries++; 161 else 162 CurrentSubEntries += getSubEntrySize(Item); 163 updateMaxStackSize(); 164 } 165 166 void CFStack::pushLoop() { 167 LoopStack.push_back(CFStack::ENTRY); 168 CurrentEntries++; 169 updateMaxStackSize(); 170 } 171 172 void CFStack::popBranch() { 173 CFStack::StackItem Top = BranchStack.back(); 174 if (Top == CFStack::ENTRY) 175 CurrentEntries--; 176 else 177 CurrentSubEntries-= getSubEntrySize(Top); 178 BranchStack.pop_back(); 179 } 180 181 void CFStack::popLoop() { 182 CurrentEntries--; 183 LoopStack.pop_back(); 184 } 185 186 class R600ControlFlowFinalizer : public MachineFunctionPass { 187 private: 188 using ClauseFile = std::pair<MachineInstr *, std::vector<MachineInstr *>>; 189 190 enum ControlFlowInstruction { 191 CF_TC, 192 CF_VC, 193 CF_CALL_FS, 194 CF_WHILE_LOOP, 195 CF_END_LOOP, 196 CF_LOOP_BREAK, 197 CF_LOOP_CONTINUE, 198 CF_JUMP, 199 CF_ELSE, 200 CF_POP, 201 CF_END 202 }; 203 204 const R600InstrInfo *TII = nullptr; 205 const R600RegisterInfo *TRI = nullptr; 206 unsigned MaxFetchInst; 207 const R600Subtarget *ST = nullptr; 208 209 bool IsTrivialInst(MachineInstr &MI) const { 210 switch (MI.getOpcode()) { 211 case R600::KILL: 212 case R600::RETURN: 213 return true; 214 default: 215 return false; 216 } 217 } 218 219 const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const { 220 unsigned Opcode = 0; 221 bool isEg = (ST->getGeneration() >= AMDGPUSubtarget::EVERGREEN); 222 switch (CFI) { 223 case CF_TC: 224 Opcode = isEg ? R600::CF_TC_EG : R600::CF_TC_R600; 225 break; 226 case CF_VC: 227 Opcode = isEg ? R600::CF_VC_EG : R600::CF_VC_R600; 228 break; 229 case CF_CALL_FS: 230 Opcode = isEg ? R600::CF_CALL_FS_EG : R600::CF_CALL_FS_R600; 231 break; 232 case CF_WHILE_LOOP: 233 Opcode = isEg ? R600::WHILE_LOOP_EG : R600::WHILE_LOOP_R600; 234 break; 235 case CF_END_LOOP: 236 Opcode = isEg ? R600::END_LOOP_EG : R600::END_LOOP_R600; 237 break; 238 case CF_LOOP_BREAK: 239 Opcode = isEg ? R600::LOOP_BREAK_EG : R600::LOOP_BREAK_R600; 240 break; 241 case CF_LOOP_CONTINUE: 242 Opcode = isEg ? R600::CF_CONTINUE_EG : R600::CF_CONTINUE_R600; 243 break; 244 case CF_JUMP: 245 Opcode = isEg ? R600::CF_JUMP_EG : R600::CF_JUMP_R600; 246 break; 247 case CF_ELSE: 248 Opcode = isEg ? R600::CF_ELSE_EG : R600::CF_ELSE_R600; 249 break; 250 case CF_POP: 251 Opcode = isEg ? R600::POP_EG : R600::POP_R600; 252 break; 253 case CF_END: 254 if (ST->hasCaymanISA()) { 255 Opcode = R600::CF_END_CM; 256 break; 257 } 258 Opcode = isEg ? R600::CF_END_EG : R600::CF_END_R600; 259 break; 260 } 261 assert (Opcode && "No opcode selected"); 262 return TII->get(Opcode); 263 } 264 265 bool isCompatibleWithClause(const MachineInstr &MI, 266 std::set<unsigned> &DstRegs) const { 267 unsigned DstMI, SrcMI; 268 for (MachineInstr::const_mop_iterator I = MI.operands_begin(), 269 E = MI.operands_end(); 270 I != E; ++I) { 271 const MachineOperand &MO = *I; 272 if (!MO.isReg()) 273 continue; 274 if (MO.isDef()) { 275 Register Reg = MO.getReg(); 276 if (R600::R600_Reg128RegClass.contains(Reg)) 277 DstMI = Reg; 278 else 279 DstMI = TRI->getMatchingSuperReg(Reg, 280 R600RegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)), 281 &R600::R600_Reg128RegClass); 282 } 283 if (MO.isUse()) { 284 Register Reg = MO.getReg(); 285 if (R600::R600_Reg128RegClass.contains(Reg)) 286 SrcMI = Reg; 287 else 288 SrcMI = TRI->getMatchingSuperReg(Reg, 289 R600RegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)), 290 &R600::R600_Reg128RegClass); 291 } 292 } 293 if ((DstRegs.find(SrcMI) == DstRegs.end())) { 294 DstRegs.insert(DstMI); 295 return true; 296 } else 297 return false; 298 } 299 300 ClauseFile 301 MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I) 302 const { 303 MachineBasicBlock::iterator ClauseHead = I; 304 std::vector<MachineInstr *> ClauseContent; 305 unsigned AluInstCount = 0; 306 bool IsTex = TII->usesTextureCache(*ClauseHead); 307 std::set<unsigned> DstRegs; 308 for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) { 309 if (IsTrivialInst(*I)) 310 continue; 311 if (AluInstCount >= MaxFetchInst) 312 break; 313 if ((IsTex && !TII->usesTextureCache(*I)) || 314 (!IsTex && !TII->usesVertexCache(*I))) 315 break; 316 if (!isCompatibleWithClause(*I, DstRegs)) 317 break; 318 AluInstCount ++; 319 ClauseContent.push_back(&*I); 320 } 321 MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead), 322 getHWInstrDesc(IsTex?CF_TC:CF_VC)) 323 .addImm(0) // ADDR 324 .addImm(AluInstCount - 1); // COUNT 325 return ClauseFile(MIb, std::move(ClauseContent)); 326 } 327 328 void getLiteral(MachineInstr &MI, std::vector<MachineOperand *> &Lits) const { 329 static const unsigned LiteralRegs[] = { 330 R600::ALU_LITERAL_X, 331 R600::ALU_LITERAL_Y, 332 R600::ALU_LITERAL_Z, 333 R600::ALU_LITERAL_W 334 }; 335 const SmallVector<std::pair<MachineOperand *, int64_t>, 3> Srcs = 336 TII->getSrcs(MI); 337 for (const auto &Src:Srcs) { 338 if (Src.first->getReg() != R600::ALU_LITERAL_X) 339 continue; 340 int64_t Imm = Src.second; 341 std::vector<MachineOperand *>::iterator It = 342 llvm::find_if(Lits, [&](MachineOperand *val) { 343 return val->isImm() && (val->getImm() == Imm); 344 }); 345 346 // Get corresponding Operand 347 MachineOperand &Operand = MI.getOperand( 348 TII->getOperandIdx(MI.getOpcode(), R600::OpName::literal)); 349 350 if (It != Lits.end()) { 351 // Reuse existing literal reg 352 unsigned Index = It - Lits.begin(); 353 Src.first->setReg(LiteralRegs[Index]); 354 } else { 355 // Allocate new literal reg 356 assert(Lits.size() < 4 && "Too many literals in Instruction Group"); 357 Src.first->setReg(LiteralRegs[Lits.size()]); 358 Lits.push_back(&Operand); 359 } 360 } 361 } 362 363 MachineBasicBlock::iterator insertLiterals( 364 MachineBasicBlock::iterator InsertPos, 365 const std::vector<unsigned> &Literals) const { 366 MachineBasicBlock *MBB = InsertPos->getParent(); 367 for (unsigned i = 0, e = Literals.size(); i < e; i+=2) { 368 unsigned LiteralPair0 = Literals[i]; 369 unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0; 370 InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(), 371 TII->get(R600::LITERALS)) 372 .addImm(LiteralPair0) 373 .addImm(LiteralPair1); 374 } 375 return InsertPos; 376 } 377 378 ClauseFile 379 MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I) 380 const { 381 MachineInstr &ClauseHead = *I; 382 std::vector<MachineInstr *> ClauseContent; 383 I++; 384 for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) { 385 if (IsTrivialInst(*I)) { 386 ++I; 387 continue; 388 } 389 if (!I->isBundle() && !TII->isALUInstr(I->getOpcode())) 390 break; 391 std::vector<MachineOperand *>Literals; 392 if (I->isBundle()) { 393 MachineInstr &DeleteMI = *I; 394 MachineBasicBlock::instr_iterator BI = I.getInstrIterator(); 395 while (++BI != E && BI->isBundledWithPred()) { 396 BI->unbundleFromPred(); 397 for (MachineOperand &MO : BI->operands()) { 398 if (MO.isReg() && MO.isInternalRead()) 399 MO.setIsInternalRead(false); 400 } 401 getLiteral(*BI, Literals); 402 ClauseContent.push_back(&*BI); 403 } 404 I = BI; 405 DeleteMI.eraseFromParent(); 406 } else { 407 getLiteral(*I, Literals); 408 ClauseContent.push_back(&*I); 409 I++; 410 } 411 for (unsigned i = 0, e = Literals.size(); i < e; i += 2) { 412 MachineInstrBuilder MILit = BuildMI(MBB, I, I->getDebugLoc(), 413 TII->get(R600::LITERALS)); 414 if (Literals[i]->isImm()) { 415 MILit.addImm(Literals[i]->getImm()); 416 } else { 417 MILit.addGlobalAddress(Literals[i]->getGlobal(), 418 Literals[i]->getOffset()); 419 } 420 if (i + 1 < e) { 421 if (Literals[i + 1]->isImm()) { 422 MILit.addImm(Literals[i + 1]->getImm()); 423 } else { 424 MILit.addGlobalAddress(Literals[i + 1]->getGlobal(), 425 Literals[i + 1]->getOffset()); 426 } 427 } else 428 MILit.addImm(0); 429 ClauseContent.push_back(MILit); 430 } 431 } 432 assert(ClauseContent.size() < 128 && "ALU clause is too big"); 433 ClauseHead.getOperand(7).setImm(ClauseContent.size() - 1); 434 return ClauseFile(&ClauseHead, std::move(ClauseContent)); 435 } 436 437 void EmitFetchClause(MachineBasicBlock::iterator InsertPos, 438 const DebugLoc &DL, ClauseFile &Clause, 439 unsigned &CfCount) { 440 CounterPropagateAddr(*Clause.first, CfCount); 441 MachineBasicBlock *BB = Clause.first->getParent(); 442 BuildMI(BB, DL, TII->get(R600::FETCH_CLAUSE)).addImm(CfCount); 443 for (MachineInstr *MI : Clause.second) 444 BB->splice(InsertPos, BB, MI); 445 CfCount += 2 * Clause.second.size(); 446 } 447 448 void EmitALUClause(MachineBasicBlock::iterator InsertPos, const DebugLoc &DL, 449 ClauseFile &Clause, unsigned &CfCount) { 450 Clause.first->getOperand(0).setImm(0); 451 CounterPropagateAddr(*Clause.first, CfCount); 452 MachineBasicBlock *BB = Clause.first->getParent(); 453 BuildMI(BB, DL, TII->get(R600::ALU_CLAUSE)).addImm(CfCount); 454 for (MachineInstr *MI : Clause.second) 455 BB->splice(InsertPos, BB, MI); 456 CfCount += Clause.second.size(); 457 } 458 459 void CounterPropagateAddr(MachineInstr &MI, unsigned Addr) const { 460 MI.getOperand(0).setImm(Addr + MI.getOperand(0).getImm()); 461 } 462 void CounterPropagateAddr(const std::set<MachineInstr *> &MIs, 463 unsigned Addr) const { 464 for (MachineInstr *MI : MIs) { 465 CounterPropagateAddr(*MI, Addr); 466 } 467 } 468 469 public: 470 static char ID; 471 472 R600ControlFlowFinalizer() : MachineFunctionPass(ID) {} 473 474 bool runOnMachineFunction(MachineFunction &MF) override { 475 ST = &MF.getSubtarget<R600Subtarget>(); 476 MaxFetchInst = ST->getTexVTXClauseSize(); 477 TII = ST->getInstrInfo(); 478 TRI = ST->getRegisterInfo(); 479 480 R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>(); 481 482 CFStack CFStack(ST, MF.getFunction().getCallingConv()); 483 for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME; 484 ++MB) { 485 MachineBasicBlock &MBB = *MB; 486 unsigned CfCount = 0; 487 std::vector<std::pair<unsigned, std::set<MachineInstr *>>> LoopStack; 488 std::vector<MachineInstr * > IfThenElseStack; 489 if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_VS) { 490 BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), 491 getHWInstrDesc(CF_CALL_FS)); 492 CfCount++; 493 } 494 std::vector<ClauseFile> FetchClauses, AluClauses; 495 std::vector<MachineInstr *> LastAlu(1); 496 std::vector<MachineInstr *> ToPopAfter; 497 498 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); 499 I != E;) { 500 if (TII->usesTextureCache(*I) || TII->usesVertexCache(*I)) { 501 LLVM_DEBUG(dbgs() << CfCount << ":"; I->dump();); 502 FetchClauses.push_back(MakeFetchClause(MBB, I)); 503 CfCount++; 504 LastAlu.back() = nullptr; 505 continue; 506 } 507 508 MachineBasicBlock::iterator MI = I; 509 if (MI->getOpcode() != R600::ENDIF) 510 LastAlu.back() = nullptr; 511 if (MI->getOpcode() == R600::CF_ALU) 512 LastAlu.back() = &*MI; 513 I++; 514 bool RequiresWorkAround = 515 CFStack.requiresWorkAroundForInst(MI->getOpcode()); 516 switch (MI->getOpcode()) { 517 case R600::CF_ALU_PUSH_BEFORE: 518 if (RequiresWorkAround) { 519 LLVM_DEBUG(dbgs() 520 << "Applying bug work-around for ALU_PUSH_BEFORE\n"); 521 BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(R600::CF_PUSH_EG)) 522 .addImm(CfCount + 1) 523 .addImm(1); 524 MI->setDesc(TII->get(R600::CF_ALU)); 525 CfCount++; 526 CFStack.pushBranch(R600::CF_PUSH_EG); 527 } else 528 CFStack.pushBranch(R600::CF_ALU_PUSH_BEFORE); 529 LLVM_FALLTHROUGH; 530 case R600::CF_ALU: 531 I = MI; 532 AluClauses.push_back(MakeALUClause(MBB, I)); 533 LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump();); 534 CfCount++; 535 break; 536 case R600::WHILELOOP: { 537 CFStack.pushLoop(); 538 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 539 getHWInstrDesc(CF_WHILE_LOOP)) 540 .addImm(1); 541 std::pair<unsigned, std::set<MachineInstr *>> Pair(CfCount, 542 std::set<MachineInstr *>()); 543 Pair.second.insert(MIb); 544 LoopStack.push_back(std::move(Pair)); 545 MI->eraseFromParent(); 546 CfCount++; 547 break; 548 } 549 case R600::ENDLOOP: { 550 CFStack.popLoop(); 551 std::pair<unsigned, std::set<MachineInstr *>> Pair = 552 std::move(LoopStack.back()); 553 LoopStack.pop_back(); 554 CounterPropagateAddr(Pair.second, CfCount); 555 BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP)) 556 .addImm(Pair.first + 1); 557 MI->eraseFromParent(); 558 CfCount++; 559 break; 560 } 561 case R600::IF_PREDICATE_SET: { 562 LastAlu.push_back(nullptr); 563 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 564 getHWInstrDesc(CF_JUMP)) 565 .addImm(0) 566 .addImm(0); 567 IfThenElseStack.push_back(MIb); 568 LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump();); 569 MI->eraseFromParent(); 570 CfCount++; 571 break; 572 } 573 case R600::ELSE: { 574 MachineInstr * JumpInst = IfThenElseStack.back(); 575 IfThenElseStack.pop_back(); 576 CounterPropagateAddr(*JumpInst, CfCount); 577 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 578 getHWInstrDesc(CF_ELSE)) 579 .addImm(0) 580 .addImm(0); 581 LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump();); 582 IfThenElseStack.push_back(MIb); 583 MI->eraseFromParent(); 584 CfCount++; 585 break; 586 } 587 case R600::ENDIF: { 588 CFStack.popBranch(); 589 if (LastAlu.back()) { 590 ToPopAfter.push_back(LastAlu.back()); 591 } else { 592 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 593 getHWInstrDesc(CF_POP)) 594 .addImm(CfCount + 1) 595 .addImm(1); 596 (void)MIb; 597 LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump();); 598 CfCount++; 599 } 600 601 MachineInstr *IfOrElseInst = IfThenElseStack.back(); 602 IfThenElseStack.pop_back(); 603 CounterPropagateAddr(*IfOrElseInst, CfCount); 604 IfOrElseInst->getOperand(1).setImm(1); 605 LastAlu.pop_back(); 606 MI->eraseFromParent(); 607 break; 608 } 609 case R600::BREAK: { 610 CfCount ++; 611 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 612 getHWInstrDesc(CF_LOOP_BREAK)) 613 .addImm(0); 614 LoopStack.back().second.insert(MIb); 615 MI->eraseFromParent(); 616 break; 617 } 618 case R600::CONTINUE: { 619 MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI), 620 getHWInstrDesc(CF_LOOP_CONTINUE)) 621 .addImm(0); 622 LoopStack.back().second.insert(MIb); 623 MI->eraseFromParent(); 624 CfCount++; 625 break; 626 } 627 case R600::RETURN: { 628 DebugLoc DL = MBB.findDebugLoc(MI); 629 BuildMI(MBB, MI, DL, getHWInstrDesc(CF_END)); 630 CfCount++; 631 if (CfCount % 2) { 632 BuildMI(MBB, I, DL, TII->get(R600::PAD)); 633 CfCount++; 634 } 635 MI->eraseFromParent(); 636 for (ClauseFile &CF : FetchClauses) 637 EmitFetchClause(I, DL, CF, CfCount); 638 for (ClauseFile &CF : AluClauses) 639 EmitALUClause(I, DL, CF, CfCount); 640 break; 641 } 642 default: 643 if (TII->isExport(MI->getOpcode())) { 644 LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump();); 645 CfCount++; 646 } 647 break; 648 } 649 } 650 for (MachineInstr *Alu : ToPopAfter) { 651 BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu), 652 TII->get(R600::CF_ALU_POP_AFTER)) 653 .addImm(Alu->getOperand(0).getImm()) 654 .addImm(Alu->getOperand(1).getImm()) 655 .addImm(Alu->getOperand(2).getImm()) 656 .addImm(Alu->getOperand(3).getImm()) 657 .addImm(Alu->getOperand(4).getImm()) 658 .addImm(Alu->getOperand(5).getImm()) 659 .addImm(Alu->getOperand(6).getImm()) 660 .addImm(Alu->getOperand(7).getImm()) 661 .addImm(Alu->getOperand(8).getImm()); 662 Alu->eraseFromParent(); 663 } 664 MFI->CFStackSize = CFStack.MaxStackSize; 665 } 666 667 return false; 668 } 669 670 StringRef getPassName() const override { 671 return "R600 Control Flow Finalizer Pass"; 672 } 673 }; 674 675 } // end anonymous namespace 676 677 INITIALIZE_PASS_BEGIN(R600ControlFlowFinalizer, DEBUG_TYPE, 678 "R600 Control Flow Finalizer", false, false) 679 INITIALIZE_PASS_END(R600ControlFlowFinalizer, DEBUG_TYPE, 680 "R600 Control Flow Finalizer", false, false) 681 682 char R600ControlFlowFinalizer::ID = 0; 683 684 char &llvm::R600ControlFlowFinalizerID = R600ControlFlowFinalizer::ID; 685 686 FunctionPass *llvm::createR600ControlFlowFinalizer() { 687 return new R600ControlFlowFinalizer(); 688 } 689