Lines Matching full:fi
468 static bool checkPHIs(FlattenInfo &FI, const TargetTransformInfo *TTI) { in checkPHIs() argument
484 SafeOuterPHIs.insert(FI.OuterInductionPHI); in checkPHIs()
488 for (PHINode &InnerPHI : FI.InnerLoop->getHeader()->phis()) { in checkPHIs()
491 if (&InnerPHI == FI.InnerInductionPHI) in checkPHIs()
493 if (FI.isNarrowInductionPhi(&InnerPHI)) in checkPHIs()
500 InnerPHI.getIncomingValueForBlock(FI.InnerLoop->getLoopPreheader()); in checkPHIs()
502 InnerPHI.getIncomingValueForBlock(FI.InnerLoop->getLoopLatch()); in checkPHIs()
508 if (!OuterPHI || OuterPHI->getParent() != FI.OuterLoop->getHeader()) { in checkPHIs()
518 OuterPHI->getIncomingValueForBlock(FI.OuterLoop->getLoopLatch())); in checkPHIs()
536 FI.InnerPHIsToTransform.insert(&InnerPHI); in checkPHIs()
539 for (PHINode &OuterPHI : FI.OuterLoop->getHeader()->phis()) { in checkPHIs()
540 if (FI.isNarrowInductionPhi(&OuterPHI)) in checkPHIs()
553 checkOuterLoopInsts(FlattenInfo &FI, in checkOuterLoopInsts() argument
562 for (auto *B : FI.OuterLoop->getBlocks()) { in checkOuterLoopInsts()
563 if (FI.InnerLoop->contains(B)) in checkOuterLoopInsts()
584 Br->getSuccessor(0) == FI.InnerLoop->getHeader()) in checkOuterLoopInsts()
588 if (match(&I, m_c_Mul(m_Specific(FI.OuterInductionPHI), in checkOuterLoopInsts()
589 m_Specific(FI.InnerTripCount)))) in checkOuterLoopInsts()
620 static bool checkIVUsers(FlattenInfo &FI) { in checkIVUsers() argument
624 if (!FI.checkInnerInductionPhiUsers(ValidOuterPHIUses)) in checkIVUsers()
629 if (!FI.checkOuterInductionPhiUsers(ValidOuterPHIUses)) in checkIVUsers()
633 dbgs() << "Found " << FI.LinearIVUses.size() in checkIVUsers()
635 for (Value *V : FI.LinearIVUses) { in checkIVUsers()
644 static OverflowResult checkOverflow(FlattenInfo &FI, DominatorTree *DT, in checkOverflow() argument
646 Function *F = FI.OuterLoop->getHeader()->getParent(); in checkOverflow()
656 FI.InnerTripCount, FI.OuterTripCount, in checkOverflow()
658 FI.OuterLoop->getLoopPreheader()->getTerminator())); in checkOverflow()
668 if (!isGuaranteedToExecuteForEveryIteration(GEPUserInst, FI.InnerLoop)) in checkOverflow()
688 for (Value *V : FI.LinearIVUses) { in checkOverflow()
701 static bool CanFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI, in CanFlattenLoopPair() argument
705 if (!findLoopComponents(FI.InnerLoop, IterationInstructions, in CanFlattenLoopPair()
706 FI.InnerInductionPHI, FI.InnerTripCount, in CanFlattenLoopPair()
707 FI.InnerIncrement, FI.InnerBranch, SE, FI.Widened)) in CanFlattenLoopPair()
709 if (!findLoopComponents(FI.OuterLoop, IterationInstructions, in CanFlattenLoopPair()
710 FI.OuterInductionPHI, FI.OuterTripCount, in CanFlattenLoopPair()
711 FI.OuterIncrement, FI.OuterBranch, SE, FI.Widened)) in CanFlattenLoopPair()
716 if (!FI.OuterLoop->isLoopInvariant(FI.InnerTripCount)) { in CanFlattenLoopPair()
720 if (!FI.OuterLoop->isLoopInvariant(FI.OuterTripCount)) { in CanFlattenLoopPair()
725 if (!checkPHIs(FI, TTI)) in CanFlattenLoopPair()
729 if (FI.InnerInductionPHI->getType() != FI.OuterInductionPHI->getType()) in CanFlattenLoopPair()
732 if (!checkOuterLoopInsts(FI, IterationInstructions, TTI)) in CanFlattenLoopPair()
740 if (!checkIVUsers(FI)) in CanFlattenLoopPair()
747 static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI, in DoFlattenLoopPair() argument
751 Function *F = FI.OuterLoop->getHeader()->getParent(); in DoFlattenLoopPair()
755 OptimizationRemark Remark(DEBUG_TYPE, "Flattened", FI.InnerLoop->getStartLoc(), in DoFlattenLoopPair()
756 FI.InnerLoop->getHeader()); in DoFlattenLoopPair()
762 if (!FI.NewTripCount) { in DoFlattenLoopPair()
763 FI.NewTripCount = BinaryOperator::CreateMul( in DoFlattenLoopPair()
764 FI.InnerTripCount, FI.OuterTripCount, "flatten.tripcount", in DoFlattenLoopPair()
765 FI.OuterLoop->getLoopPreheader()->getTerminator()->getIterator()); in DoFlattenLoopPair()
767 FI.NewTripCount->dump()); in DoFlattenLoopPair()
772 FI.InnerInductionPHI->removeIncomingValue(FI.InnerLoop->getLoopLatch()); in DoFlattenLoopPair()
776 for (PHINode *PHI : FI.InnerPHIsToTransform) in DoFlattenLoopPair()
777 PHI->removeIncomingValue(FI.InnerLoop->getLoopLatch()); in DoFlattenLoopPair()
781 cast<User>(FI.OuterBranch->getCondition())->setOperand(1, FI.NewTripCount); in DoFlattenLoopPair()
784 BasicBlock *InnerExitBlock = FI.InnerLoop->getExitBlock(); in DoFlattenLoopPair()
785 BasicBlock *InnerExitingBlock = FI.InnerLoop->getExitingBlock(); in DoFlattenLoopPair()
792 DT->deleteEdge(InnerExitingBlock, FI.InnerLoop->getHeader()); in DoFlattenLoopPair()
794 MSSAU->removeEdge(InnerExitingBlock, FI.InnerLoop->getHeader()); in DoFlattenLoopPair()
798 IRBuilder<> Builder(FI.OuterInductionPHI->getParent()->getTerminator()); in DoFlattenLoopPair()
799 for (Value *V : FI.LinearIVUses) { in DoFlattenLoopPair()
800 Value *OuterValue = FI.OuterInductionPHI; in DoFlattenLoopPair()
801 if (FI.Widened) in DoFlattenLoopPair()
802 OuterValue = Builder.CreateTrunc(FI.OuterInductionPHI, V->getType(), in DoFlattenLoopPair()
826 SE->forgetLoop(FI.OuterLoop); in DoFlattenLoopPair()
829 U->markLoopAsDeleted(*FI.InnerLoop, FI.InnerLoop->getName()); in DoFlattenLoopPair()
830 LI->erase(FI.InnerLoop); in DoFlattenLoopPair()
838 static bool CanWidenIV(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI, in CanWidenIV() argument
847 Module *M = FI.InnerLoop->getHeader()->getParent()->getParent(); in CanWidenIV()
849 auto *InnerType = FI.InnerInductionPHI->getType(); in CanWidenIV()
850 auto *OuterType = FI.OuterInductionPHI->getType(); in CanWidenIV()
883 if (!CreateWideIV({FI.InnerInductionPHI, MaxLegalType, false}, Deleted)) in CanWidenIV()
888 FI.InnerPHIsToTransform.insert(FI.InnerInductionPHI); in CanWidenIV()
890 if (!CreateWideIV({FI.OuterInductionPHI, MaxLegalType, false}, Deleted)) in CanWidenIV()
894 FI.Widened = true; in CanWidenIV()
897 FI.NarrowInnerInductionPHI = FI.InnerInductionPHI; in CanWidenIV()
898 FI.NarrowOuterInductionPHI = FI.OuterInductionPHI; in CanWidenIV()
901 return CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI); in CanWidenIV()
904 static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI, in FlattenLoopPair() argument
911 << FI.OuterLoop->getHeader()->getName() << " and inner loop " in FlattenLoopPair()
912 << FI.InnerLoop->getHeader()->getName() << " in " in FlattenLoopPair()
913 << FI.OuterLoop->getHeader()->getParent()->getName() << "\n"); in FlattenLoopPair()
915 if (!CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI)) in FlattenLoopPair()
919 bool CanFlatten = CanWidenIV(FI, DT, LI, SE, AC, TTI); in FlattenLoopPair()
932 if (FI.Widened && !CanFlatten) in FlattenLoopPair()
937 return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI, U, MSSAU); in FlattenLoopPair()
943 OverflowResult OR = checkOverflow(FI, DT, AC); in FlattenLoopPair()
949 Module *M = FI.OuterLoop->getHeader()->getParent()->getParent(); in FlattenLoopPair()
955 FI.OuterTripCount->getType()->getScalarSizeInBits())) { in FlattenLoopPair()
968 BasicBlock *CheckBlock = FI.OuterLoop->getLoopPreheader(); in FlattenLoopPair()
970 LoopVersioning LVer(LAI, Checks, FI.OuterLoop, LI, DT, SE); in FlattenLoopPair()
982 FI.OuterTripCount->getType()); in FlattenLoopPair()
983 Value *Call = Builder.CreateCall(F, {FI.OuterTripCount, FI.InnerTripCount}, in FlattenLoopPair()
985 FI.NewTripCount = Builder.CreateExtractValue(Call, 0, "flatten.tripcount"); in FlattenLoopPair()
992 return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI, U, MSSAU); in FlattenLoopPair()
1017 FlattenInfo FI(OuterLoop, InnerLoop); in run() local
1019 FlattenLoopPair(FI, &AR.DT, &AR.LI, &AR.SE, &AR.AC, &AR.TTI, &U, in run()