Lines Matching +full:vcc +full:- +full:p
1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
16 /// by writing to the 64-bit EXEC register (each bit corresponds to a
18 /// to its bit of the VCC register (like EXEC VCC is 64-bits, one for each
19 /// Vector ALU) and then the ScalarALU will AND the VCC register with the
23 /// %vcc = V_CMP_GT_F32 %vgpr1, %vgpr2
24 /// %sgpr0 = SI_IF %vcc
32 /// %sgpr0 = S_AND_SAVEEXEC_B64 %vcc // Save and update the exec mask
48 /// %exec = S_OR_B64 %exec, %sgpr0 // Re-enable saved exec mask bits
49 //===----------------------------------------------------------------------===//
63 #define DEBUG_TYPE "si-lower-control-flow"
66 RemoveRedundantEndcf("amdgpu-remove-redundant-endcf",
126 assert(I->isTerminator()); in skipToUncondBrOrEnd()
128 // FIXME: What if we had multiple pre-existing conditional branches? in skipToUncondBrOrEnd()
130 while (I != End && !I->isUnconditionalBranch()) in skipToUncondBrOrEnd()
179 SmallVector<MachineBasicBlock *, 4> Worklist(Begin->successors()); in hasKill()
189 Worklist.append(MBB->succ_begin(), MBB->succ_end()); in hasKill()
197 auto U = MRI->use_instr_nodbg_begin(SaveExecReg); in isSimpleIf()
199 if (U == MRI->use_instr_nodbg_end() || in isSimpleIf()
200 std::next(U) != MRI->use_instr_nodbg_end() || in isSimpleIf()
201 U->getOpcode() != AMDGPU::SI_END_CF) in isSimpleIf()
226 auto UseMI = MRI->use_instr_nodbg_begin(SaveExecReg); in emitIf()
227 SimpleIf = !hasKill(MI.getParent(), UseMI->getParent()); in emitIf()
233 : MRI->createVirtualRegister(BoolRC); in emitIf()
235 BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), CopyReg) in emitIf()
240 Register Tmp = MRI->createVirtualRegister(BoolRC); in emitIf()
243 BuildMI(MBB, I, DL, TII->get(AndOpc), Tmp) in emitIf()
247 LV->replaceKillInstruction(Cond.getReg(), MI, *And); in emitIf()
254 BuildMI(MBB, I, DL, TII->get(XorOpc), SaveExecReg) in emitIf()
263 BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec) in emitIf()
266 LV->getVarInfo(Tmp).Kills.push_back(SetExec); in emitIf()
274 MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) in emitIf()
282 LIS->InsertMachineInstrInMaps(*CopyExec); in emitIf()
286 LIS->ReplaceMachineInstrInMaps(MI, *And); in emitIf()
289 LIS->InsertMachineInstrInMaps(*Xor); in emitIf()
290 LIS->InsertMachineInstrInMaps(*SetExec); in emitIf()
291 LIS->InsertMachineInstrInMaps(*NewBr); in emitIf()
293 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC); in emitIf()
300 LIS->createAndComputeVirtRegInterval(Tmp); in emitIf()
302 LIS->createAndComputeVirtRegInterval(CopyReg); in emitIf()
316 Register SaveReg = MRI->createVirtualRegister(BoolRC); in emitElse()
318 BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg) in emitElse()
321 LV->replaceKillInstruction(SrcReg, MI, *OrSaveExec); in emitElse()
328 // can be optimized out pre-RA when not required. in emitElse()
329 MachineInstr *And = BuildMI(MBB, ElsePt, DL, TII->get(AndOpc), DstReg) in emitElse()
334 BuildMI(MBB, ElsePt, DL, TII->get(XorTermrOpc), Exec) in emitElse()
343 BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) in emitElse()
351 LIS->RemoveMachineInstrFromMaps(MI); in emitElse()
354 LIS->InsertMachineInstrInMaps(*OrSaveExec); in emitElse()
355 LIS->InsertMachineInstrInMaps(*And); in emitElse()
357 LIS->InsertMachineInstrInMaps(*Xor); in emitElse()
358 LIS->InsertMachineInstrInMaps(*Branch); in emitElse()
362 LIS->createAndComputeVirtRegInterval(SaveReg); in emitElse()
365 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC); in emitElse()
376 // be one with a carry-out.) in emitIfBreak()
379 if (MachineInstr *Def = MRI->getUniqueVRegDef(MI.getOperand(1).getReg())) { in emitIfBreak()
380 SkipAnding = Def->getParent() == MI.getParent() in emitIfBreak()
390 AndReg = MRI->createVirtualRegister(BoolRC); in emitIfBreak()
391 And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg) in emitIfBreak()
395 LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *And); in emitIfBreak()
396 Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) in emitIfBreak()
400 Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) in emitIfBreak()
404 LV->replaceKillInstruction(MI.getOperand(1).getReg(), MI, *Or); in emitIfBreak()
407 LV->replaceKillInstruction(MI.getOperand(2).getReg(), MI, *Or); in emitIfBreak()
410 LIS->ReplaceMachineInstrInMaps(MI, *Or); in emitIfBreak()
413 RecomputeRegs.insert(And->getOperand(2).getReg()); in emitIfBreak()
414 LIS->InsertMachineInstrInMaps(*And); in emitIfBreak()
415 LIS->createAndComputeVirtRegInterval(AndReg); in emitIfBreak()
427 BuildMI(MBB, &MI, DL, TII->get(Andn2TermOpc), Exec) in emitLoop()
431 LV->replaceKillInstruction(MI.getOperand(0).getReg(), MI, *AndN2); in emitLoop()
435 BuildMI(MBB, BranchPt, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ)) in emitLoop()
440 LIS->ReplaceMachineInstrInMaps(MI, *AndN2); in emitLoop()
441 LIS->InsertMachineInstrInMaps(*Branch); in emitLoop()
457 auto E = B->end(); in skipIgnoreExecInstsTrivialSucc()
459 if (TII->mayReadEXEC(*MRI, *It)) in skipIgnoreExecInstsTrivialSucc()
466 if (B->succ_size() != 1) in skipIgnoreExecInstsTrivialSucc()
470 MachineBasicBlock *Succ = *B->succ_begin(); in skipIgnoreExecInstsTrivialSucc()
472 It = Succ->begin(); in skipIgnoreExecInstsTrivialSucc()
490 if (I->modifiesRegister(DataReg, TRI)) { in emitEndCf()
502 SmallVector<MachineDomTreeNode *> Children(MBBNode->begin(), in emitEndCf()
503 MBBNode->end()); in emitEndCf()
504 MachineDomTreeNode *SplitBBNode = MDT->addNewBlock(SplitBB, &MBB); in emitEndCf()
506 MDT->changeImmediateDominator(Child, SplitBBNode); in emitEndCf()
513 BuildMI(MBB, InsPt, DL, TII->get(Opcode), Exec) in emitEndCf()
517 LV->replaceKillInstruction(DataReg, MI, *NewMI); in emitEndCf()
535 for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) { in emitEndCf()
537 LiveVariables::VarInfo &VI = LV->getVarInfo(Reg); in emitEndCf()
540 VI.AliveBlocks.set(SplitBB->getNumber()); in emitEndCf()
543 if (Kill->getParent() == SplitBB && !DefInOrigBlock.contains(Reg)) in emitEndCf()
554 LIS->ReplaceMachineInstrInMaps(MI, *NewMI); in emitEndCf()
559 LIS->handleMove(*NewMI); in emitEndCf()
573 MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg()); in findMaskOperands()
574 if (!Def || Def->getParent() != MI.getParent() || in findMaskOperands()
575 !(Def->isFullCopy() || (Def->getOpcode() == MI.getOpcode()))) in findMaskOperands()
581 for (auto I = Def->getIterator(); I != MI.getIterator(); ++I) in findMaskOperands()
582 if (I->modifiesRegister(AMDGPU::EXEC, TRI) && in findMaskOperands()
583 !(I->isCopy() && I->getOperand(0).getReg() != Exec)) in findMaskOperands()
586 for (const auto &SrcOp : Def->explicit_operands()) in findMaskOperands()
614 if (MRI->use_empty(Reg)) in combineMasks()
615 MRI->getUniqueVRegDef(Reg)->eraseFromParent(); in combineMasks()
625 MachineBasicBlock &MBB = *MI->getParent(); in optimizeEndCf()
627 skipIgnoreExecInstsTrivialSucc(MBB, std::next(MI->getIterator())); in optimizeEndCf()
633 = TII->getNamedOperand(*Next, AMDGPU::OpName::src1)->getReg(); in optimizeEndCf()
636 const MachineInstr *Def = MRI->getUniqueVRegDef(SavedExec); in optimizeEndCf()
638 LLVM_DEBUG(dbgs() << "Skip redundant "; MI->dump()); in optimizeEndCf()
640 LIS->RemoveMachineInstrFromMaps(*MI); in optimizeEndCf()
643 Reg = TII->getNamedOperand(*MI, AMDGPU::OpName::src1)->getReg(); in optimizeEndCf()
644 MI->eraseFromParent(); in optimizeEndCf()
646 LV->recomputeForSingleDefVirtReg(Reg); in optimizeEndCf()
677 MI.setDesc(TII->get(AMDGPU::S_CBRANCH_EXECNZ)); in process()
690 for (I = Prev ? Prev->getIterator() : MBB.begin(); I != MBB.end(); I = Next) { in process()
722 MachineBasicBlock *P = *MBB.pred_begin(); in removeMBBifRedundant() local
723 if (P->getFallThrough(false) == &MBB) in removeMBBifRedundant()
724 FallThrough = P; in removeMBBifRedundant()
725 P->ReplaceUsesOfBlockWith(&MBB, Succ); in removeMBBifRedundant()
730 LIS->RemoveMachineInstrFromMaps(I); in removeMBBifRedundant()
736 if (MDT->dominates(&MBB, Succ)) in removeMBBifRedundant()
737 MDT->changeImmediateDominator(MDT->getNode(Succ), in removeMBBifRedundant()
738 MDT->getNode(&MBB)->getIDom()); in removeMBBifRedundant()
739 MDT->eraseNode(&MBB); in removeMBBifRedundant()
743 if (FallThrough && !FallThrough->isLayoutSuccessor(Succ)) { in removeMBBifRedundant()
746 MachineInstr *BranchMI = BuildMI(*FallThrough, FallThrough->end(), in removeMBBifRedundant()
747 FallThrough->findBranchDebugLoc(), TII->get(AMDGPU::S_BRANCH)) in removeMBBifRedundant()
750 LIS->InsertMachineInstrInMaps(*BranchMI); in removeMBBifRedundant()
759 TRI = &TII->getRegisterInfo(); in runOnMachineFunction()
765 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr; in runOnMachineFunction()
768 LV = LVWrapper ? &LVWrapper->getLV() : nullptr; in runOnMachineFunction()
770 MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr; in runOnMachineFunction()
772 BoolRC = TRI->getBoolRC(); in runOnMachineFunction()
802 if (TII->isKillTerminator(Term.getOpcode())) { in runOnMachineFunction()
826 E = MBB->end(); in runOnMachineFunction()
827 for (I = MBB->begin(); I != E; I = Next) { in runOnMachineFunction()
845 MBB = Next->getParent(); in runOnMachineFunction()
846 E = MBB->end(); in runOnMachineFunction()
855 LIS->removeInterval(Reg); in runOnMachineFunction()
856 LIS->createAndComputeVirtRegInterval(Reg); in runOnMachineFunction()