Lines Matching full:region

77   /// replace the extracted function for each region.
101 /// extraction of the first region.
112 /// The number of branches in the region target a basic block that is outside
113 /// of the region.
138 /// we check whether it is the same in Region.
266 // region is the same as the recorded instruction following the last in splitCandidate()
281 // We iterate over the instructions in the region, if we find a PHINode, we in splitCandidate()
282 // check if there are predecessors outside of the region, if there are, in splitCandidate()
283 // we ignore this region since we are unable to handle the severing of the in splitCandidate()
318 // If the region starts with a PHINode, but is not the initial instruction of in splitCandidate()
319 // the BasicBlock, we ignore this region for now. in splitCandidate()
323 // If the region ends with a PHINode, but does not contain all of the phi node in splitCandidate()
324 // instructions of the region, we ignore it for now. in splitCandidate()
348 // If there was a PHINode with an incoming block outside the region, in splitCandidate()
368 // For the phi nodes in the new starting basic block of the region, we in splitCandidate()
405 // incoming blocks to the first block are contained in the region, and there in reattachCandidate()
526 /// after being used as an output for an outlined region.
539 /// Find whether \p Region matches the global value numbering to Constant
542 /// \param Region - The OutlinableRegion we are checking for constants
545 /// constant in each region.
547 /// Region and false if not
549 collectRegionsConstants(OutlinableRegion &Region, in collectRegionsConstants() argument
554 IRSimilarityCandidate &C = *Region.Candidate; in collectRegionsConstants()
601 for (OutlinableRegion *Region : Regions) in findSameConstants()
602 collectRegionsConstants(*Region, GVNToConstant, NotSame); in findSameConstants()
636 // region occur in the same place. in createFunction()
639 // needed, meaning that we could branch within the region or out, it is in createFunction()
772 /// as they are not the same in each instance of the region.
774 /// \param [in] C - The IRSimilarityCandidate containing the region we are
779 /// arguments needed for the region of code.
802 /// \param [in] C - The IRSimilarityCandidate containing the region we are
847 /// Find the input GVNs and the output values for a region of Instructions.
850 /// The \p Region can be identified as needing to be ignored in this function.
854 /// \param [in,out] Region - The region of code to be analyzed.
857 /// \param [in] NotSame - The global value numbers in the region that do not
859 /// \p Region.
866 OutlinableRegion &Region, std::vector<unsigned> &InputGVNs, in getCodeExtractorArguments() argument
869 IRSimilarityCandidate &C = *Region.Candidate; in getCodeExtractorArguments()
871 // OverallInputs are the inputs to the region found by the CodeExtractor, in getCodeExtractorArguments()
874 // outlined region. PremappedInputs are the arguments found by the in getCodeExtractorArguments()
885 CodeExtractor *CE = Region.CE; in getCodeExtractorArguments()
887 assert(Region.StartBB && "Region must have a start BasicBlock!"); in getCodeExtractorArguments()
888 Function *OrigF = Region.StartBB->getParent(); in getCodeExtractorArguments()
892 // The region may be ineligible due to VarArgs in the parent function. In this in getCodeExtractorArguments()
893 // case we ignore the region. in getCodeExtractorArguments()
895 Region.IgnoreRegion = true; in getCodeExtractorArguments()
904 // contained completely within the outlined region. These are not guaranteed in getCodeExtractorArguments()
905 // to be the same in every region, so we must elevate them all to arguments in getCodeExtractorArguments()
909 Region.IgnoreRegion = true; in getCodeExtractorArguments()
930 /// \param [in,out] Region - The region of code to be analyzed.
936 findExtractedInputToOverallInputMapping(OutlinableRegion &Region, in findExtractedInputToOverallInputMapping() argument
940 IRSimilarityCandidate &C = *Region.Candidate; in findExtractedInputToOverallInputMapping()
941 OutlinableGroup &Group = *Region.Parent; in findExtractedInputToOverallInputMapping()
983 // number to Constant map for the region, and continue to the next input. in findExtractedInputToOverallInputMapping()
986 Region.AggArgToConstant.insert(std::make_pair(AggArgIt->second, CST)); in findExtractedInputToOverallInputMapping()
990 Region.AggArgToConstant.insert(std::make_pair(TypeIndex, CST)); in findExtractedInputToOverallInputMapping()
1002 Region.ChangedArgOrder = true; in findExtractedInputToOverallInputMapping()
1003 Region.ExtractedArgToAgg.insert( in findExtractedInputToOverallInputMapping()
1005 Region.AggArgToExtracted.insert( in findExtractedInputToOverallInputMapping()
1010 Region.ExtractedArgToAgg.insert(std::make_pair(OriginalIndex, TypeIndex)); in findExtractedInputToOverallInputMapping()
1011 Region.AggArgToExtracted.insert(std::make_pair(TypeIndex, OriginalIndex)); in findExtractedInputToOverallInputMapping()
1017 // If the function type definitions for the OutlinableGroup holding the region in findExtractedInputToOverallInputMapping()
1026 Region.NumExtractedInputs = OriginalIndex; in findExtractedInputToOverallInputMapping()
1029 /// Check if the \p V has any uses outside of the region other than \p PN.
1035 /// region.
1036 /// \param BlocksInRegion [in] - The basic blocks contained in the region.
1037 /// \returns true if \p V has any use soutside its region other than \p PN.
1042 // predecessor not included in the region. If it is, we make sure in outputHasNonPHI()
1051 // Check if the value is used by any other instructions outside the region. in outputHasNonPHI()
1057 // If the use of the item is inside the region, we skip it. Uses in outputHasNonPHI()
1058 // inside the region give us useful information about how the item could be in outputHasNonPHI()
1087 /// region.
1088 /// \param RegionBlocks [in] - The basic blocks in the region.
1089 /// \param Outputs [in, out] - The existing outputs for the region, we may add
1102 // Find all incoming values from the outlining region. in analyzeExitPHIsForOutputUses()
1108 // Do not process PHI if there are no predecessors from region. in analyzeExitPHIsForOutputUses()
1126 // outputs may have uses in outlined region. If they have other uses in analyzeExitPHIsForOutputUses()
1163 /// the region. We create a hash code based on the Canonical number of the
1170 /// \param Region - The region that \p PN is an output for.
1172 /// \param Blocks - The blocks for the region we are analyzing.
1176 static std::optional<unsigned> getGVNForPHINode(OutlinableRegion &Region, in getGVNForPHINode() argument
1180 OutlinableGroup &Group = *Region.Parent; in getGVNForPHINode()
1181 IRSimilarityCandidate &Cand = *Region.Candidate; in getGVNForPHINode()
1189 // If we cannot find a GVN, and the incoming block is included in the region in getGVNForPHINode()
1190 // this means that the input to the PHINode is not included in the region we in getGVNForPHINode()
1193 // region. in getGVNForPHINode()
1196 Region.IgnoreRegion = true; in getGVNForPHINode()
1200 // If the incoming block isn't in the region, we don't have to worry about in getGVNForPHINode()
1224 // PHINode, when we find a block that is not contained in the region in getGVNForPHINode()
1272 /// Create a mapping of the output arguments for the \p Region to the output
1275 /// \param [in,out] Region - The region of code to be analyzed.
1278 findExtractedOutputToOverallOutputMapping(Module &M, OutlinableRegion &Region, in findExtractedOutputToOverallOutputMapping() argument
1280 OutlinableGroup &Group = *Region.Parent; in findExtractedOutputToOverallOutputMapping()
1281 IRSimilarityCandidate &C = *Region.Candidate; in findExtractedOutputToOverallOutputMapping()
1287 // Find the exits to the region. in findExtractedOutputToOverallOutputMapping()
1305 unsigned OriginalIndex = Region.NumExtractedInputs; in findExtractedOutputToOverallOutputMapping()
1341 Region.ExtractedArgToAgg.insert(std::make_pair(OriginalIndex, Jdx)); in findExtractedOutputToOverallOutputMapping()
1342 Region.AggArgToExtracted.insert(std::make_pair(Jdx, OriginalIndex)); in findExtractedOutputToOverallOutputMapping()
1357 Region.ExtractedArgToAgg.insert( in findExtractedOutputToOverallOutputMapping()
1359 Region.AggArgToExtracted.insert( in findExtractedOutputToOverallOutputMapping()
1369 // Values outside the region can be combined into PHINode when we in findExtractedOutputToOverallOutputMapping()
1378 GVN = getGVNForPHINode(Region, PN, BlocksInRegion, AggArgIdx); in findExtractedOutputToOverallOutputMapping()
1389 // Each region has a potentially unique set of outputs. We save which in findExtractedOutputToOverallOutputMapping()
1392 Region.GVNStores.push_back(*GVN); in findExtractedOutputToOverallOutputMapping()
1400 stable_sort(Region.GVNStores); in findExtractedOutputToOverallOutputMapping()
1403 void IROutliner::findAddInputsOutputs(Module &M, OutlinableRegion &Region, in findAddInputsOutputs() argument
1408 getCodeExtractorArguments(Region, Inputs, NotSame, OutputMappings, ArgInputs, in findAddInputsOutputs()
1411 if (Region.IgnoreRegion) in findAddInputsOutputs()
1416 findExtractedInputToOverallInputMapping(Region, Inputs, ArgInputs); in findAddInputsOutputs()
1420 findExtractedOutputToOverallOutputMapping(M, Region, Outputs); in findAddInputsOutputs()
1423 /// Replace the extracted function in the Region with a call to the overall
1429 /// \param [in] Region - The regions of extracted code to be replaced with a new
1432 CallInst *replaceCalledFunction(Module &M, OutlinableRegion &Region) { in replaceCalledFunction() argument
1436 OutlinableGroup &Group = *Region.Parent; in replaceCalledFunction()
1437 CallInst *Call = Region.Call; in replaceCalledFunction()
1446 if (!Region.ChangedArgOrder && AggFunc->arg_size() == Call->arg_size()) { in replaceCalledFunction()
1465 << Region.OutputBlockNum << "\n"); in replaceCalledFunction()
1467 Region.OutputBlockNum)); in replaceCalledFunction()
1471 ArgPair = Region.AggArgToExtracted.find(AggArgIdx); in replaceCalledFunction()
1472 if (ArgPair != Region.AggArgToExtracted.end()) { in replaceCalledFunction()
1484 if (Region.AggArgToConstant.contains(AggArgIdx)) { in replaceCalledFunction()
1485 Constant *CST = Region.AggArgToConstant.find(AggArgIdx)->second; in replaceCalledFunction()
1494 // for the region, so we should not pass anything to it. in replaceCalledFunction()
1510 CallInst *OldCall = Region.Call; in replaceCalledFunction()
1511 if (Region.NewFront->Inst == OldCall) in replaceCalledFunction()
1512 Region.NewFront->Inst = Call; in replaceCalledFunction()
1513 if (Region.NewBack->Inst == OldCall) in replaceCalledFunction()
1514 Region.NewBack->Inst = Call; in replaceCalledFunction()
1517 Call->setDebugLoc(Region.Call->getDebugLoc()); in replaceCalledFunction()
1524 Region.Call = Call; in replaceCalledFunction()
1587 /// For the function call now representing the \p Region, find the passed value
1593 /// \param Region - The extracted Region corresponding to the outlined function.
1597 const OutlinableRegion &Region) { in getPassedArgumentInAlreadyOutlinedFunction() argument
1601 return Region.Call->getArgOperand(A->getArgNo()); in getPassedArgumentInAlreadyOutlinedFunction()
1604 /// For the function call now representing the \p Region, find the passed value
1609 /// \param Region - The extracted Region corresponding to the outlined function.
1613 const OutlinableRegion &Region) { in getPassedArgumentAndAdjustArgumentLocation() argument
1618 if (Region.AggArgToConstant.count(ArgNum)) in getPassedArgumentAndAdjustArgumentLocation()
1619 return Region.AggArgToConstant.find(ArgNum)->second; in getPassedArgumentAndAdjustArgumentLocation()
1623 ArgNum = Region.AggArgToExtracted.find(ArgNum)->second; in getPassedArgumentAndAdjustArgumentLocation()
1624 return Region.Call->getArgOperand(ArgNum); in getPassedArgumentAndAdjustArgumentLocation()
1630 /// \param Region [in] - The OutlinableRegion containing \p PN.
1632 /// region to their original values.
1636 /// of \p Region rather than the overall function's call.
1638 PHINode *PN, OutlinableRegion &Region, in findCanonNumsForPHI() argument
1650 IVal = getPassedArgumentInAlreadyOutlinedFunction(A, Region); in findCanonNumsForPHI()
1652 IVal = getPassedArgumentAndAdjustArgumentLocation(A, Region); in findCanonNumsForPHI()
1659 std::optional<unsigned> GVN = Region.Candidate->getGVN(IVal); in findCanonNumsForPHI()
1661 std::optional<unsigned> CanonNum = Region.Candidate->getCanonicalNum(*GVN); in findCanonNumsForPHI()
1672 /// \param Region [in] - The OutlinableRegion containing \p PN.
1676 /// region to their original values.
1681 findOrCreatePHIInBlock(PHINode &PN, OutlinableRegion &Region, in findOrCreatePHIInBlock() argument
1685 OutlinableGroup &Group = *Region.Parent; in findOrCreatePHIInBlock()
1692 // We have to use the extracted function since we have merged this region into in findOrCreatePHIInBlock()
1696 findCanonNumsForPHI(&PN, Region, OutputMappings, PNCanonNums, in findOrCreatePHIInBlock()
1726 // in PHINode to one already present in the outlined region. If the in findOrCreatePHIInBlock()
1730 // finding the corresponding incoming block in the combined outlined region in findOrCreatePHIInBlock()
1731 // for the current outlined region. in findOrCreatePHIInBlock()
1741 Region.findCorrespondingBlockIn(*FirstRegion, ToAdd.second); in findOrCreatePHIInBlock()
1769 Region.findCorrespondingBlockIn(*FirstRegion, IncomingBlock); in findOrCreatePHIInBlock()
1782 Value *Val = Region.findCorrespondingValueIn(*FirstRegion, IncomingVal); in findOrCreatePHIInBlock()
1794 // region with the arguments of the function for an OutlinableGroup.
1796 /// \param [in] Region - The region of extracted code to be changed.
1798 /// region.
1803 replaceArgumentUses(OutlinableRegion &Region, in replaceArgumentUses() argument
1807 OutlinableGroup &Group = *Region.Parent; in replaceArgumentUses()
1808 assert(Region.ExtractedFunction && "Region has no extracted function?"); in replaceArgumentUses()
1810 Function *DominatingFunction = Region.ExtractedFunction; in replaceArgumentUses()
1816 for (unsigned ArgIdx = 0; ArgIdx < Region.ExtractedFunction->arg_size(); in replaceArgumentUses()
1818 assert(Region.ExtractedArgToAgg.contains(ArgIdx) && in replaceArgumentUses()
1820 unsigned AggArgIdx = Region.ExtractedArgToAgg.find(ArgIdx)->second; in replaceArgumentUses()
1822 Argument *Arg = Region.ExtractedFunction->getArg(ArgIdx); in replaceArgumentUses()
1825 if (ArgIdx < Region.NumExtractedInputs) { in replaceArgumentUses()
1827 << *Region.ExtractedFunction << " with " << *AggArg in replaceArgumentUses()
1830 Value *V = Region.Call->getArgOperand(ArgIdx); in replaceArgumentUses()
1831 Region.RemappedArguments.insert(std::make_pair(V, AggArg)); in replaceArgumentUses()
1880 Region.Candidate->getGVN(ValueOperand).has_value()) { in replaceArgumentUses()
1884 Region.findCorrespondingValueIn(*Group.Regions[0], ValueOperand); in replaceArgumentUses()
1892 if (Region.Candidate->getGVN(PN)) in replaceArgumentUses()
1895 // We record the parent block for the PHINode in the Region so that in replaceArgumentUses()
1897 Region.PHIBlocks.insert(std::make_pair(RetVal, PN->getParent())); in replaceArgumentUses()
1915 PHINode *NewPN = findOrCreatePHIInBlock(*PN, Region, OverallPhiBlock, in replaceArgumentUses()
1927 << *Region.ExtractedFunction << " with " << *AggArg in replaceArgumentUses()
1936 /// \param Region [in] - The region of extracted code to be changed.
1937 void replaceConstants(OutlinableRegion &Region) { in replaceConstants() argument
1938 OutlinableGroup &Group = *Region.Parent; in replaceConstants()
1940 for (std::pair<unsigned, Constant *> &Const : Region.AggArgToConstant) { in replaceConstants()
2019 /// Remove empty output blocks from the outlined region.
2022 /// Region.
2023 /// \param Region - The OutlinableRegion we are analyzing.
2026 OutlinableRegion &Region) { in analyzeAndPruneOutputBlocks() argument
2053 // Mark the region as having the no output scheme. in analyzeAndPruneOutputBlocks()
2055 Region.OutputBlockNum = -1; in analyzeAndPruneOutputBlocks()
2065 /// \param [in] Region - The OutlinableRegion that is being analyzed.
2066 /// \param [in,out] OutputBBs - the blocks that stores for this region will be
2073 OutlinableGroup &OG, OutlinableRegion &Region, in alignOutputBlockWithAggFunc() argument
2081 if (analyzeAndPruneOutputBlocks(OutputBBs, Region)) in alignOutputBlockWithAggFunc()
2091 LLVM_DEBUG(dbgs() << "Set output block for region in function" in alignOutputBlockWithAggFunc()
2092 << Region.ExtractedFunction << " to " << *MatchingBB); in alignOutputBlockWithAggFunc()
2094 Region.OutputBlockNum = *MatchingBB; in alignOutputBlockWithAggFunc()
2100 Region.OutputBlockNum = OutputStoreBBs.size(); in alignOutputBlockWithAggFunc()
2110 LLVM_DEBUG(dbgs() << "Create output block for region in" in alignOutputBlockWithAggFunc()
2111 << Region.ExtractedFunction << " to " in alignOutputBlockWithAggFunc()
2160 // in a PHINode outside of the region in one outlined instance, and are used in createSwitchStatement()
2234 /// the extracted regions of a certain structure from the first region in the
2235 /// list of regions. Replace this first region's extracted function with the
2364 const OutlinableRegion &Region) { in isCompatibleWithAlreadyOutlinedCode() argument
2365 IRSimilarityCandidate *IRSC = Region.Candidate; in isCompatibleWithAlreadyOutlinedCode()
2377 if (!Region.Candidate->backInstruction()->isTerminator()) { in isCompatibleWithAlreadyOutlinedCode()
2379 Region.Candidate->backInstruction()->getNextNonDebugInstruction(); in isCompatibleWithAlreadyOutlinedCode()
2381 if (Region.Candidate->end()->Inst != NewEndInst) { in isCompatibleWithAlreadyOutlinedCode()
2382 IRInstructionDataList *IDL = Region.Candidate->front()->IDL; in isCompatibleWithAlreadyOutlinedCode()
2387 // Insert the first IRInstructionData of the new region after the in isCompatibleWithAlreadyOutlinedCode()
2389 IDL->insert(Region.Candidate->end(), *NewEndIRID); in isCompatibleWithAlreadyOutlinedCode()
2487 for (OutlinableRegion *Region : CurrentGroup.Regions) { in findBenefitFromAllRegions()
2488 TargetTransformInfo &TTI = getTTI(*Region->StartBB->getParent()); in findBenefitFromAllRegions()
2489 // We add the number of instructions in the region to the benefit as an in findBenefitFromAllRegions()
2491 RegionBenefit += Region->getBenefit(TTI); in findBenefitFromAllRegions()
2503 /// \param Region - The OutlinableRegion to get the Value from.
2506 /// Region.
2507 static Value *findOutputValueInRegion(OutlinableRegion &Region, in findOutputValueInRegion() argument
2509 OutlinableGroup &CurrentGroup = *Region.Parent; in findOutputValueInRegion()
2521 Region.Candidate->fromCanonicalNum(OutputCanon); in findOutputValueInRegion()
2523 std::optional<Value *> OV = Region.Candidate->fromGVN(*OGVN); in findOutputValueInRegion()
2531 for (OutlinableRegion *Region : CurrentGroup.Regions) { in findCostOutputReloads()
2532 TargetTransformInfo &TTI = getTTI(*Region->StartBB->getParent()); in findCostOutputReloads()
2535 for (unsigned OutputCanon : Region->GVNStores) { in findCostOutputReloads()
2536 Value *V = findOutputValueInRegion(*Region, OutputCanon); in findCostOutputReloads()
2552 /// region.
2571 // of the region. in findCostForOutputBlocks()
2648 // We add one region to the cost once, to account for the instructions added in findCostBenefit()
2679 void IROutliner::updateOutputMapping(OutlinableRegion &Region, in updateOutputMapping() argument
2686 for (unsigned ArgIdx = Region.NumExtractedInputs; in updateOutputMapping()
2687 ArgIdx < Region.Call->arg_size(); ArgIdx++) { in updateOutputMapping()
2688 if (Operand == Region.Call->getArgOperand(ArgIdx)) { in updateOutputMapping()
2689 OutputIdx = ArgIdx - Region.NumExtractedInputs; in updateOutputMapping()
2711 bool IROutliner::extractSection(OutlinableRegion &Region) { in extractSection() argument
2713 assert(Region.StartBB && "StartBB for the OutlinableRegion is nullptr!"); in extractSection()
2714 BasicBlock *InitialStart = Region.StartBB; in extractSection()
2715 Function *OrigF = Region.StartBB->getParent(); in extractSection()
2717 Region.ExtractedFunction = in extractSection()
2718 Region.CE->extractCodeRegion(CEAC, ArgInputs, Outputs); in extractSection()
2722 if (!Region.ExtractedFunction) { in extractSection()
2723 LLVM_DEBUG(dbgs() << "CodeExtractor failed to outline " << Region.StartBB in extractSection()
2725 Region.reattachCandidate(); in extractSection()
2733 User *InstAsUser = Region.ExtractedFunction->user_back(); in extractSection()
2735 Region.PrevBB = RewrittenBB->getSinglePredecessor(); in extractSection()
2736 assert(Region.PrevBB && "PrevBB is nullptr?"); in extractSection()
2737 if (Region.PrevBB == InitialStart) { in extractSection()
2742 Region.PrevBB = NewPrev; in extractSection()
2746 Region.StartBB = RewrittenBB; in extractSection()
2747 Region.EndBB = RewrittenBB; in extractSection()
2754 IRInstructionDataList *IDL = Region.Candidate->front()->IDL; in extractSection()
2757 Region.NewFront = new (InstDataAllocator.Allocate()) IRInstructionData( in extractSection()
2759 Region.NewBack = new (InstDataAllocator.Allocate()) IRInstructionData( in extractSection()
2762 // Insert the first IRInstructionData of the new region in front of the in extractSection()
2764 IDL->insert(Region.Candidate->begin(), *Region.NewFront); in extractSection()
2765 // Insert the first IRInstructionData of the new region after the in extractSection()
2767 IDL->insert(Region.Candidate->end(), *Region.NewBack); in extractSection()
2769 IDL->erase(Region.Candidate->begin(), std::prev(Region.Candidate->end())); in extractSection()
2778 if (Region.ExtractedFunction == CI->getCalledFunction()) in extractSection()
2779 Region.Call = CI; in extractSection()
2781 updateOutputMapping(Region, Outputs.getArrayRef(), LI); in extractSection()
2782 Region.reattachCandidate(); in extractSection()
2835 // Create a CodeExtractor for each outlinable region. Identify inputs and in doOutline()
2840 // Break the outlinable region out of its parent BasicBlock into its own in doOutline()
2844 // There's a chance that when the region is split, extra instructions are in doOutline()
2845 // added to the region. This makes the region no longer viable in doOutline()
2892 [&R](OutlinableRegion *Region) { in doOutline() argument
2895 Region->Candidate->frontInstruction()->getDebugLoc()); in doOutline()
2919 for (OutlinableRegion *Region : CurrentGroup.Regions) { in doOutline()
2920 // We check whether our region is compatible with what has already been in doOutline()
2922 if (!isCompatibleWithAlreadyOutlinedCode(*Region)) in doOutline()
2924 OutlinedRegions.push_back(Region); in doOutline()
2941 for (OutlinableRegion *Region : CurrentGroup.Regions) { in doOutline()
2942 Region->splitCandidate(); in doOutline()
2943 if (!Region->CandidateSplit) in doOutline()
2945 OutlinedRegions.push_back(Region); in doOutline()
2998 [&R](OutlinableRegion *Region) { in doOutline() argument
3000 Region->Candidate->frontInstruction()->getDebugLoc()); in doOutline()