Lines Matching refs:FC0
657 bool isControlFlowEquivalent(const FusionCandidate &FC0, in isControlFlowEquivalent()
659 assert(FC0.Preheader && FC1.Preheader && "Expecting valid preheaders"); in isControlFlowEquivalent()
661 return ::isControlFlowEquivalent(*FC0.getEntryBlock(), *FC1.getEntryBlock(), in isControlFlowEquivalent()
714 bool isBeneficialFusion(const FusionCandidate &FC0, in isBeneficialFusion()
728 haveIdenticalTripCounts(const FusionCandidate &FC0, in haveIdenticalTripCounts()
730 const SCEV *TripCount0 = SE.getBackedgeTakenCount(FC0.L); in haveIdenticalTripCounts()
757 const unsigned TC0 = SE.getSmallConstantTripCount(FC0.L); in haveIdenticalTripCounts()
786 void peelFusionCandidate(FusionCandidate &FC0, const FusionCandidate &FC1, in peelFusionCandidate()
788 assert(FC0.AbleToPeel && "Should be able to peel loop"); in peelFusionCandidate()
794 FC0.Peeled = peelLoop(FC0.L, PeelCount, &LI, &SE, DT, &AC, true, VMap); in peelFusionCandidate()
795 if (FC0.Peeled) { in peelFusionCandidate()
799 auto IdenticalTripCount = haveIdenticalTripCounts(FC0, FC1); in peelFusionCandidate()
805 FC0.PP.PeelCount += PeelCount; in peelFusionCandidate()
808 PDT.recalculate(*FC0.Preheader->getParent()); in peelFusionCandidate()
810 FC0.updateAfterPeeling(); in peelFusionCandidate()
821 FC0.GuardBranch ? FC0.ExitBlock->getUniqueSuccessor() : FC1.Preheader; in peelFusionCandidate()
826 if (Pred != FC0.ExitBlock) { in peelFusionCandidate()
845 dbgs() << "Sucessfully peeled " << FC0.PP.PeelCount in peelFusionCandidate()
868 for (auto FC0 = CandidateSet.begin(); FC0 != CandidateSet.end(); ++FC0) { in fuseCandidates() local
869 assert(!LDT.isRemovedLoop(FC0->L) && in fuseCandidates()
871 auto FC1 = FC0; in fuseCandidates()
876 LLVM_DEBUG(dbgs() << "Attempting to fuse candidate \n"; FC0->dump(); in fuseCandidates()
879 FC0->verify(); in fuseCandidates()
888 haveIdenticalTripCounts(*FC0, *FC1); in fuseCandidates()
894 if (FC0->AbleToPeel && !SameTripCount && TCDifference) { in fuseCandidates()
910 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
915 if (!isAdjacent(*FC0, *FC1)) { in fuseCandidates()
918 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, NonAdjacent); in fuseCandidates()
922 if ((!FC0->GuardBranch && FC1->GuardBranch) || in fuseCandidates()
923 (FC0->GuardBranch && !FC1->GuardBranch)) { in fuseCandidates()
927 *FC0, *FC1, OnlySecondCandidateIsGuarded); in fuseCandidates()
933 if (FC0->GuardBranch && FC1->GuardBranch && in fuseCandidates()
934 !haveIdenticalGuards(*FC0, *FC1) && !TCDifference) { in fuseCandidates()
937 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
942 if (FC0->GuardBranch) { in fuseCandidates()
945 if (!isSafeToMoveBefore(*FC0->ExitBlock, in fuseCandidates()
950 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
957 *FC0->GuardBranch->getParent()->getTerminator(), DT, &PDT, in fuseCandidates()
962 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
970 if (!dependencesAllowFusion(*FC0, *FC1)) { in fuseCandidates()
972 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
991 if (!collectMovablePreheaderInsts(*FC0, *FC1, SafeToHoist, in fuseCandidates()
996 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
1002 bool BeneficialToFuse = isBeneficialFusion(*FC0, *FC1); in fuseCandidates()
1007 reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1, in fuseCandidates()
1016 movePreheaderInsts(*FC0, *FC1, SafeToHoist, SafeToSink); in fuseCandidates()
1018 LLVM_DEBUG(dbgs() << "\tFusion is performed: " << *FC0 << " and " in fuseCandidates()
1021 FusionCandidate FC0Copy = *FC0; in fuseCandidates()
1032 reportLoopFusion<OptimizationRemark>((Peel ? FC0Copy : *FC0), *FC1, in fuseCandidates()
1036 performFusion((Peel ? FC0Copy : *FC0), *FC1), DT, &PDT, ORE, in fuseCandidates()
1045 CandidateSet.erase(FC0); in fuseCandidates()
1056 FC0 = FC1 = InsertPos.first; in fuseCandidates()
1076 const FusionCandidate &FC0) const { in canHoistInst()
1077 const BasicBlock *FC0PreheaderTarget = FC0.Preheader->getSingleSuccessor(); in canHoistInst()
1114 for (Instruction *ReadInst : FC0.MemReads) { in canHoistInst()
1124 for (Instruction *WriteInst : FC0.MemWrites) { in canHoistInst()
1182 const FusionCandidate &FC0, const FusionCandidate &FC1, in collectMovablePreheaderInsts()
1211 if (canHoistInst(I, SafeToHoist, NotHoisting, FC0)) { in collectMovablePreheaderInsts()
1325 bool dependencesAllowFusion(const FusionCandidate &FC0, in dependencesAllowFusion()
1337 return accessDiffIsPositive(*FC0.L, *FC1.L, I0, I1, AnyDep); in dependencesAllowFusion()
1362 return dependencesAllowFusion(FC0, FC1, I0, I1, AnyDep, in dependencesAllowFusion()
1364 dependencesAllowFusion(FC0, FC1, I0, I1, AnyDep, in dependencesAllowFusion()
1372 bool dependencesAllowFusion(const FusionCandidate &FC0, in dependencesAllowFusion()
1374 LLVM_DEBUG(dbgs() << "Check if " << FC0 << " can be fused with " << FC1 in dependencesAllowFusion()
1376 assert(FC0.L->getLoopDepth() == FC1.L->getLoopDepth()); in dependencesAllowFusion()
1377 assert(DT.dominates(FC0.getEntryBlock(), FC1.getEntryBlock())); in dependencesAllowFusion()
1379 for (Instruction *WriteL0 : FC0.MemWrites) { in dependencesAllowFusion()
1381 if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *WriteL1, in dependencesAllowFusion()
1388 if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *ReadL1, in dependencesAllowFusion()
1397 for (Instruction *WriteL0 : FC0.MemWrites) in dependencesAllowFusion()
1398 if (!dependencesAllowFusion(FC0, FC1, *WriteL0, *WriteL1, in dependencesAllowFusion()
1404 for (Instruction *ReadL0 : FC0.MemReads) in dependencesAllowFusion()
1405 if (!dependencesAllowFusion(FC0, FC1, *ReadL0, *WriteL1, in dependencesAllowFusion()
1419 if (FC0.L->contains(Def->getParent())) { in dependencesAllowFusion()
1436 bool isAdjacent(const FusionCandidate &FC0, in isAdjacent()
1439 if (FC0.GuardBranch) in isAdjacent()
1440 return FC0.getNonLoopBlock() == FC1.getEntryBlock(); in isAdjacent()
1442 return FC0.ExitBlock == FC1.getEntryBlock(); in isAdjacent()
1451 void movePreheaderInsts(const FusionCandidate &FC0, in movePreheaderInsts()
1476 I->moveBefore(*FC0.Preheader, in movePreheaderInsts()
1477 FC0.Preheader->getTerminator()->getIterator()); in movePreheaderInsts()
1498 bool haveIdenticalGuards(const FusionCandidate &FC0, in haveIdenticalGuards()
1500 assert(FC0.GuardBranch && FC1.GuardBranch && in haveIdenticalGuards()
1504 dyn_cast<Instruction>(FC0.GuardBranch->getCondition())) in haveIdenticalGuards()
1513 if (FC0.GuardBranch->getSuccessor(0) == FC0.Preheader) in haveIdenticalGuards()
1535 void mergeLatch(const FusionCandidate &FC0, const FusionCandidate &FC1) { in mergeLatch()
1536 moveInstructionsToTheBeginning(*FC0.Latch, *FC1.Latch, DT, PDT, DI); in mergeLatch()
1537 if (BasicBlock *Succ = FC0.Latch->getUniqueSuccessor()) { in mergeLatch()
1572 Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1) { in performFusion()
1573 assert(FC0.isValid() && FC1.isValid() && in performFusion()
1576 LLVM_DEBUG(dbgs() << "Fusion Candidate 0: \n"; FC0.dump(); in performFusion()
1581 moveInstructionsToTheEnd(*FC1.Preheader, *FC0.Preheader, DT, PDT, DI); in performFusion()
1586 if (FC0.GuardBranch) in performFusion()
1587 return fuseGuardedLoops(FC0, FC1); in performFusion()
1590 (FC0.Peeled ? FC0.ExitBlock->getUniqueSuccessor() : FC0.ExitBlock)); in performFusion()
1604 if (FC0.ExitingBlock != FC0.Latch) in performFusion()
1605 for (PHINode &PHI : FC0.Header->phis()) in performFusion()
1609 FC1.Preheader->replaceSuccessorsPhiUsesWith(FC0.Preheader); in performFusion()
1610 FC0.Latch->replaceSuccessorsPhiUsesWith(FC1.Latch); in performFusion()
1632 if (!FC0.Peeled) { in performFusion()
1633 FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC1.Preheader, in performFusion()
1636 DominatorTree::Delete, FC0.ExitingBlock, FC1.Preheader)); in performFusion()
1638 DominatorTree::Insert, FC0.ExitingBlock, FC1.Header)); in performFusion()
1641 DominatorTree::Delete, FC0.ExitBlock, FC1.Preheader)); in performFusion()
1644 FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC0.ExitBlock, in performFusion()
1647 DominatorTree::Delete, FC0.ExitingBlock, FC0.ExitBlock)); in performFusion()
1648 FC0.ExitBlock->getTerminator()->eraseFromParent(); in performFusion()
1650 DominatorTree::Insert, FC0.ExitingBlock, FC1.Header)); in performFusion()
1651 new UnreachableInst(FC0.ExitBlock->getContext(), FC0.ExitBlock); in performFusion()
1666 PHI->moveBefore(&*FC0.Header->getFirstInsertionPt()); in performFusion()
1686 L1HeaderPHI->addIncoming(LCV, FC0.Latch); in performFusion()
1688 FC0.ExitingBlock); in performFusion()
1694 FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header); in performFusion()
1695 FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header); in performFusion()
1699 simplifyLatchBranch(FC0); in performFusion()
1703 if (FC0.Latch != FC0.ExitingBlock) in performFusion()
1705 DominatorTree::Insert, FC0.Latch, FC1.Header)); in performFusion()
1708 FC0.Latch, FC0.Header)); in performFusion()
1710 FC1.Latch, FC0.Header)); in performFusion()
1719 if (FC0.Peeled) { in performFusion()
1720 LI.removeBlock(FC0.ExitBlock); in performFusion()
1721 DTU.deleteBB(FC0.ExitBlock); in performFusion()
1731 SE.forgetLoop(FC0.L); in performFusion()
1736 mergeLatch(FC0, FC1); in performFusion()
1741 FC0.L->addBlockEntry(BB); in performFusion()
1745 LI.changeLoopFor(BB, FC0.L); in performFusion()
1751 FC0.L->addChildLoop(ChildLoop); in performFusion()
1758 assert(!verifyFunction(*FC0.Header->getParent(), &errs())); in performFusion()
1767 return FC0.L; in performFusion()
1783 void reportLoopFusion(const FusionCandidate &FC0, const FusionCandidate &FC1, in reportLoopFusion()
1785 assert(FC0.Preheader && FC1.Preheader && in reportLoopFusion()
1790 ORE.emit(RemarkKind(DEBUG_TYPE, Stat.getName(), FC0.L->getStartLoc(), in reportLoopFusion()
1791 FC0.Preheader) in reportLoopFusion()
1792 << "[" << FC0.Preheader->getParent()->getName() in reportLoopFusion()
1793 << "]: " << NV("Cand1", StringRef(FC0.Preheader->getName())) in reportLoopFusion()
1814 Loop *fuseGuardedLoops(const FusionCandidate &FC0, in fuseGuardedLoops()
1816 assert(FC0.GuardBranch && FC1.GuardBranch && "Expecting guarded loops"); in fuseGuardedLoops()
1818 BasicBlock *FC0GuardBlock = FC0.GuardBranch->getParent(); in fuseGuardedLoops()
1820 BasicBlock *FC0NonLoopBlock = FC0.getNonLoopBlock(); in fuseGuardedLoops()
1822 BasicBlock *FC0ExitBlockSuccessor = FC0.ExitBlock->getUniqueSuccessor(); in fuseGuardedLoops()
1829 (FC0.Peeled ? *FC0ExitBlockSuccessor : *FC0.ExitBlock), *FC1.ExitBlock, in fuseGuardedLoops()
1849 FC0.GuardBranch->replaceUsesOfWith(FC0NonLoopBlock, FC1NonLoopBlock); in fuseGuardedLoops()
1851 BasicBlock *BBToUpdate = FC0.Peeled ? FC0ExitBlockSuccessor : FC0.ExitBlock; in fuseGuardedLoops()
1867 if (FC0.Peeled) { in fuseGuardedLoops()
1894 if (FC0.ExitingBlock != FC0.Latch) in fuseGuardedLoops()
1895 for (PHINode &PHI : FC0.Header->phis()) in fuseGuardedLoops()
1901 FC1.Preheader->replaceSuccessorsPhiUsesWith(FC0.Preheader); in fuseGuardedLoops()
1902 FC0.Latch->replaceSuccessorsPhiUsesWith(FC1.Latch); in fuseGuardedLoops()
1917 FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC0.ExitBlock, in fuseGuardedLoops()
1921 DominatorTree::Delete, FC0.ExitingBlock, FC0.ExitBlock)); in fuseGuardedLoops()
1923 DominatorTree::Insert, FC0.ExitingBlock, FC1.Header)); in fuseGuardedLoops()
1932 assert(pred_empty(FC0.ExitBlock) && "Expecting exit block to be empty"); in fuseGuardedLoops()
1933 FC0.ExitBlock->getTerminator()->eraseFromParent(); in fuseGuardedLoops()
1934 new UnreachableInst(FC0.ExitBlock->getContext(), FC0.ExitBlock); in fuseGuardedLoops()
1949 PHI->moveBefore(&*FC0.Header->getFirstInsertionPt()); in fuseGuardedLoops()
1969 L1HeaderPHI->addIncoming(LCV, FC0.Latch); in fuseGuardedLoops()
1971 FC0.ExitingBlock); in fuseGuardedLoops()
1979 FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header); in fuseGuardedLoops()
1980 FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header); in fuseGuardedLoops()
1984 simplifyLatchBranch(FC0); in fuseGuardedLoops()
1988 if (FC0.Latch != FC0.ExitingBlock) in fuseGuardedLoops()
1990 DominatorTree::Insert, FC0.Latch, FC1.Header)); in fuseGuardedLoops()
1993 FC0.Latch, FC0.Header)); in fuseGuardedLoops()
1995 FC1.Latch, FC0.Header)); in fuseGuardedLoops()
2010 LI.removeBlock(FC0.ExitBlock); in fuseGuardedLoops()
2011 if (FC0.Peeled) { in fuseGuardedLoops()
2017 DTU.deleteBB(FC0.ExitBlock); in fuseGuardedLoops()
2025 SE.forgetLoop(FC0.L); in fuseGuardedLoops()
2030 mergeLatch(FC0, FC1); in fuseGuardedLoops()
2035 FC0.L->addBlockEntry(BB); in fuseGuardedLoops()
2039 LI.changeLoopFor(BB, FC0.L); in fuseGuardedLoops()
2045 FC0.L->addChildLoop(ChildLoop); in fuseGuardedLoops()
2052 assert(!verifyFunction(*FC0.Header->getParent(), &errs())); in fuseGuardedLoops()
2061 return FC0.L; in fuseGuardedLoops()