Lines Matching +full:int +full:- +full:threshold
1 //===- InlineCost.cpp - Cost analysis for inliner -------------------------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
31 #include "llvm/Config/llvm-config.h"
52 #define DEBUG_TYPE "inline-cost"
56 static cl::opt<int>
57 DefaultThreshold("inlinedefault-threshold", cl::Hidden, cl::init(225),
60 // We introduce this option since there is a minor compile-time win by avoiding
61 // addition of TTI attributes (target-features in particular) to inline
66 "ignore-tti-inline-compatible", cl::Hidden, cl::init(false),
71 "print-instruction-comments", cl::Hidden, cl::init(false),
74 static cl::opt<int> InlineThreshold(
75 "inline-threshold", cl::Hidden, cl::init(225),
78 static cl::opt<int> HintThreshold(
79 "inlinehint-threshold", cl::Hidden, cl::init(325),
80 cl::desc("Threshold for inlining functions with inline hint"));
82 static cl::opt<int>
83 ColdCallSiteThreshold("inline-cold-callsite-threshold", cl::Hidden,
85 cl::desc("Threshold for inlining cold callsites"));
88 "inline-enable-cost-benefit-analysis", cl::Hidden, cl::init(false),
89 cl::desc("Enable the cost-benefit analysis for the inliner"));
94 static cl::opt<int> InlineSavingsMultiplier(
95 "inline-savings-multiplier", cl::Hidden, cl::init(8),
101 static cl::opt<int> InlineSavingsProfitableMultiplier(
102 "inline-savings-profitable-multiplier", cl::Hidden, cl::init(4),
106 static cl::opt<int>
107 InlineSizeAllowance("inline-size-allowance", cl::Hidden, cl::init(100),
111 // We introduce this threshold to help performance of instrumentation based
114 static cl::opt<int> ColdThreshold(
115 "inlinecold-threshold", cl::Hidden, cl::init(45),
116 cl::desc("Threshold for inlining functions with cold attribute"));
118 static cl::opt<int>
119 HotCallSiteThreshold("hot-callsite-threshold", cl::Hidden, cl::init(3000),
120 cl::desc("Threshold for hot callsites "));
122 static cl::opt<int> LocallyHotCallSiteThreshold(
123 "locally-hot-callsite-threshold", cl::Hidden, cl::init(525),
124 cl::desc("Threshold for locally hot callsites "));
126 static cl::opt<int> ColdCallSiteRelFreq(
127 "cold-callsite-rel-freq", cl::Hidden, cl::init(2),
133 "hot-callsite-rel-freq", cl::Hidden, cl::init(60),
138 static cl::opt<int>
139 InstrCost("inline-instr-cost", cl::Hidden, cl::init(5),
142 static cl::opt<int>
143 MemAccessCost("inline-memaccess-cost", cl::Hidden, cl::init(0),
146 static cl::opt<int> CallPenalty(
147 "inline-call-penalty", cl::Hidden, cl::init(25),
151 StackSizeThreshold("inline-max-stacksize", cl::Hidden,
157 "recursive-inline-max-stacksize", cl::Hidden,
163 "inline-cost-full", cl::Hidden,
165 "exceeds the threshold."));
168 "inline-caller-superset-nobuiltin", cl::Hidden, cl::init(true),
173 "disable-gep-const-evaluation", cl::Hidden, cl::init(false),
177 std::optional<int> getStringFnAttrAsInt(const Attribute &Attr) { in getStringFnAttrAsInt()
179 int AttrValue = 0; in getStringFnAttrAsInt()
186 std::optional<int> getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind) { in getStringFnAttrAsInt()
190 std::optional<int> getStringFnAttrAsInt(Function *F, StringRef AttrKind) { in getStringFnAttrAsInt()
191 return getStringFnAttrAsInt(F->getFnAttribute(AttrKind)); in getStringFnAttrAsInt()
195 int getInstrCost() { return InstrCost; } in getInstrCost()
207 int CostBefore = 0;
208 int CostAfter = 0;
209 int ThresholdBefore = 0;
210 int ThresholdAfter = 0;
212 int getThresholdDelta() const { return ThresholdAfter - ThresholdBefore; } in getThresholdDelta()
214 int getCostDelta() const { return CostAfter - CostBefore; } in getCostDelta()
231 /// as llvm::getInlineCost. Note the function_ref constructor parameters - the
293 /// contexts propagated). It checks callsite-specific information. Return a
379 /// While we walk the potentially-inlined instructions, we build up and
383 /// likely simplifications post-inlining. The most important aspect we track
384 /// is CFG altering simplifications -- when we prove a basic block dead, that
417 if (It == SROAArgValues.end() || EnabledSROAAllocas.count(It->second) == 0) in getSROAArgForValueOrNull()
419 return It->second; in getSROAArgForValueOrNull()
528 // f(n) = 1 + f(n/2) + f (n - n/2), when n > 3,
532 // the number of non-leaf should be :
533 // 2^(log2(n) - 1) - 1
534 // = 2^log2(n) * 2^-1 - 1
535 // = n / 2 - 1.
536 // Considering comparisons from leaf and non-leaf nodes, we can estimate the
538 // n + n / 2 - 1 = n * 3 / 2 - 1
539 int64_t getExpectedNumberOfCompare(int NumCaseCluster) { in getExpectedNumberOfCompare()
540 return 3 * static_cast<int64_t>(NumCaseCluster) / 2 - 1; in getExpectedNumberOfCompare()
547 int LoadEliminationCost = 0;
550 int VectorBonus = 0;
552 int SingleBBBonus = 0;
557 // This DenseMap stores the delta change in cost and threshold after
564 int Threshold = 0; member in __anon463f3bdb0111::InlineCostCallAnalyzer
567 int StaticBonusApplied = 0;
572 /// Ignore the threshold when finalizing analysis.
575 // True if the cost-benefit-analysis-based inliner is enabled.
580 /// Instructions that are statically proven to be dead based on call-site
582 int Cost = 0;
585 // the end of analyzing each basic block, "Cost - CostAtBBStart" represents
587 int CostAtBBStart = 0;
591 int ColdSize = 0;
593 // Whether inlining is decided by cost-threshold analysis.
596 // Whether inlining is decided by cost-benefit analysis.
599 // The cost-benefit pair computed by cost-benefit analysis.
610 DenseMap<AllocaInst *, int> SROAArgCosts;
615 /// Update Threshold based on callsite properties such as callee
620 /// Return a higher threshold if \p Call is a hot callsite.
621 std::optional<int> getHotCallSiteThreshold(CallBase &Call,
624 /// Handle a capped 'int' increment for Cost.
634 addCost(CostIt->second); in onDisableSROA()
635 SROACostSavings -= CostIt->second; in onDisableSROA()
636 SROACostSavingsLost += CostIt->second; in onDisableSROA()
646 if (std::optional<int> AttrCallThresholdBonus = in onCallBaseVisitStart()
647 getStringFnAttrAsInt(Call, "call-threshold-bonus")) in onCallBaseVisitStart()
648 Threshold += *AttrCallThresholdBonus; in onCallBaseVisitStart()
650 if (std::optional<int> AttrCallCost = in onCallBaseVisitStart()
651 getStringFnAttrAsInt(Call, "call-inline-cost")) { in onCallBaseVisitStart()
682 // Pretend to inline the function, with a custom threshold. in onLoweredCall()
693 // threshold to get the bonus we want to apply, but don't go below zero. in onLoweredCall()
694 Cost -= std::max(0, CA.getThreshold() - CA.getCost()); in onLoweredCall()
723 addCost((NumCaseCluster - DefaultDestUndefined) * 2 * InstrCost); in onFinalizeSwitch()
747 CostIt->second += InstrCost; in onAggregateSROAUse()
760 auto ProfileCount = BFI->getBlockProfileCount(BB); in onBlockAnalyzed()
762 ColdSize += Cost - CostAtBBStart; in onBlockAnalyzed()
765 auto *TI = BB->getTerminator(); in onBlockAnalyzed()
766 // If we had any successors at this point, than post-inlining is likely to in onBlockAnalyzed()
770 if (SingleBB && TI->getNumSuccessors() > 1) { in onBlockAnalyzed()
771 // Take off the bonus we applied to the threshold. in onBlockAnalyzed()
772 Threshold -= SingleBBBonus; in onBlockAnalyzed()
783 InstructionCostDetailMap[I].ThresholdBefore = Threshold; in onInstructionAnalysisStart()
787 // This function is called to find new values of cost and threshold after in onInstructionAnalysisFinish()
792 InstructionCostDetailMap[I].ThresholdAfter = Threshold; in onInstructionAnalysisFinish()
796 if (!PSI || !PSI->hasProfileSummary()) in isCostBenefitAnalysisEnabled()
808 if (!PSI->hasInstrumentationProfile()) in isCostBenefitAnalysisEnabled()
812 auto *Caller = CandidateCall.getParent()->getParent(); in isCostBenefitAnalysisEnabled()
813 if (!Caller->getEntryCount()) in isCostBenefitAnalysisEnabled()
821 if (!PSI->isHotCallSite(CandidateCall, CallerBFI)) in isCostBenefitAnalysisEnabled()
826 if (!EntryCount || !EntryCount->getCount()) in isCostBenefitAnalysisEnabled()
850 void OverrideCycleSavingsAndSizeForTesting(APInt &CycleSavings, int &Size) { in OverrideCycleSavingsAndSizeForTesting()
851 if (std::optional<int> AttrCycleSavings = getStringFnAttrAsInt( in OverrideCycleSavingsAndSizeForTesting()
852 CandidateCall, "inline-cycle-savings-for-test")) { in OverrideCycleSavingsAndSizeForTesting()
856 if (std::optional<int> AttrRuntimeCost = getStringFnAttrAsInt( in OverrideCycleSavingsAndSizeForTesting()
857 CandidateCall, "inline-runtime-cost-for-test")) { in OverrideCycleSavingsAndSizeForTesting()
871 // falling back to the cost-based metric. in costBenefitAnalysis()
873 if (Threshold == 0) in costBenefitAnalysis()
885 // We use 128-bit APInt here to avoid potential overflow. This variable in costBenefitAnalysis()
888 // profile count of 10^^15 -- roughly the number of cycles for a 24-hour in costBenefitAnalysis()
897 if (BI->isConditional() && in costBenefitAnalysis()
899 SimplifiedValues.lookup(BI->getCondition()))) { in costBenefitAnalysis()
903 if (isa_and_present<ConstantInt>(SimplifiedValues.lookup(SI->getCondition()))) in costBenefitAnalysis()
913 auto ProfileCount = CalleeBFI->getBlockProfileCount(&BB); in costBenefitAnalysis()
920 assert(EntryProfileCount && EntryProfileCount->getCount()); in costBenefitAnalysis()
921 auto EntryCount = EntryProfileCount->getCount(); in costBenefitAnalysis()
927 BlockFrequencyInfo *CallerBFI = &(GetBFI(*(CallerBB->getParent()))); in costBenefitAnalysis()
928 CycleSavings += getCallsiteCost(TTI, this->CandidateCall, DL); in costBenefitAnalysis()
929 CycleSavings *= *CallerBFI->getBlockProfileCount(CallerBB); in costBenefitAnalysis()
934 int Size = Cost - ColdSize; in costBenefitAnalysis()
937 // savings threshold. in costBenefitAnalysis()
938 Size = Size > InlineSizeAllowance ? Size - InlineSizeAllowance : 1; in costBenefitAnalysis()
945 // somewhere in the middle, we fall back to the cost-based analysis. in costBenefitAnalysis()
950 // PSI->getOrCompHotCountThreshold() in costBenefitAnalysis()
951 // R > ------------------------------------------------- in costBenefitAnalysis()
956 // PSI->getOrCompHotCountThreshold() in costBenefitAnalysis()
957 // R <= ---------------------------------------------------- in costBenefitAnalysis()
960 // Otherwise, we fall back to the cost-based analysis. in costBenefitAnalysis()
962 // Implementation-wise, use multiplication (CycleSavings * Multiplier, in costBenefitAnalysis()
964 APInt Threshold(128, PSI->getOrCompHotCountThreshold()); in costBenefitAnalysis() local
965 Threshold *= Size; in costBenefitAnalysis()
969 if (UpperBoundCycleSavings.uge(Threshold)) in costBenefitAnalysis()
975 if (LowerBoundCycleSavings.ult(Threshold)) in costBenefitAnalysis()
978 // Otherwise, fall back to the cost-based analysis. in costBenefitAnalysis()
989 if (Caller->hasMinSize()) { in finalizeAnalysis()
992 int NumLoops = 0; in finalizeAnalysis()
995 if (DeadBlocks.count(L->getHeader())) in finalizeAnalysis()
1003 // subtract the excess bonus, if any, from the Threshold before in finalizeAnalysis()
1006 Threshold -= VectorBonus; in finalizeAnalysis()
1008 Threshold -= VectorBonus / 2; in finalizeAnalysis()
1010 if (std::optional<int> AttrCost = in finalizeAnalysis()
1011 getStringFnAttrAsInt(CandidateCall, "function-inline-cost")) in finalizeAnalysis()
1014 if (std::optional<int> AttrCostMult = getStringFnAttrAsInt( in finalizeAnalysis()
1019 if (std::optional<int> AttrThreshold = in finalizeAnalysis()
1020 getStringFnAttrAsInt(CandidateCall, "function-inline-threshold")) in finalizeAnalysis()
1021 Threshold = *AttrThreshold; in finalizeAnalysis()
1028 return InlineResult::failure("Cost over threshold."); in finalizeAnalysis()
1035 return Cost < std::max(1, Threshold) in finalizeAnalysis()
1037 : InlineResult::failure("Cost over threshold."); in finalizeAnalysis()
1043 // Bail out the moment we cross the threshold. This means we'll under-count in shouldStop()
1045 if (Cost < Threshold) in shouldStop()
1056 // Perform some tweaks to the cost and threshold based on the direct in onAnalysisStart()
1059 // We want to more aggressively inline vector-dense kernels, so up the in onAnalysisStart()
1060 // threshold, and we'll lower it if the % of vector instructions gets too in onAnalysisStart()
1069 // Update the threshold based on callsite properties in onAnalysisStart()
1072 // While Threshold depends on commandline options that can take negative in onAnalysisStart()
1073 // values, we want to enforce the invariant that the computed threshold and in onAnalysisStart()
1074 // bonuses are non-negative. in onAnalysisStart()
1075 assert(Threshold >= 0); in onAnalysisStart()
1079 // Speculatively apply all possible bonuses to Threshold. If cost exceeds in onAnalysisStart()
1080 // this Threshold any time, and cost cannot decrease, we can stop processing in onAnalysisStart()
1082 Threshold += (SingleBBBonus + VectorBonus); in onAnalysisStart()
1086 addCost(-getCallsiteCost(TTI, this->CandidateCall, DL)); in onAnalysisStart()
1096 if (Cost >= Threshold && !ComputeFullInlineCost) in onAnalysisStart()
1115 Params(Params), Threshold(Params.DefaultThreshold), in InlineCostCallAnalyzer()
1138 int getThreshold() const { return Threshold; } in getThreshold()
1139 int getCost() const { return Cost; } in getCost()
1140 int getStaticBonusApplied() const { return StaticBonusApplied; } in getStaticBonusApplied()
1157 // FIXME: These constants are taken from the heuristic-based cost visitor.
1160 static constexpr int JTCostMultiplier = 2;
1161 static constexpr int CaseClusterCostMultiplier = 2;
1162 static constexpr int SwitchDefaultDestCostMultiplier = 2;
1163 static constexpr int SwitchCostMultiplier = 2;
1165 // FIXME: These are taken from the heuristic-based cost visitor: we should
1168 int VectorBonus = 0;
1169 int SingleBBBonus = 0;
1170 int Threshold = 5; member in __anon463f3bdb0111::InlineCostFeaturesAnalyzer
1187 increment(InlineCostFeatureIndex::sroa_losses, CostIt->second); in onDisableSROA()
1188 SROACostSavingOpportunities -= CostIt->second; in onDisableSROA()
1255 (NumCaseCluster - DefaultDestUndefined) * in onFinalizeSwitch()
1280 SROACosts.find(Arg)->second += InstrCost; in onAggregateSROAUse()
1285 if (BB->getTerminator()->getNumSuccessors() > 1) in onBlockAnalyzed()
1287 Threshold -= SingleBBBonus; in onBlockAnalyzed()
1292 if (Caller->hasMinSize()) { in finalizeAnalysis()
1297 if (DeadBlocks.count(L->getHeader())) in finalizeAnalysis()
1312 Threshold -= VectorBonus; in finalizeAnalysis()
1314 Threshold -= VectorBonus / 2; in finalizeAnalysis()
1316 set(InlineCostFeatureIndex::threshold, Threshold); in finalizeAnalysis()
1329 -1 * getCallsiteCost(TTI, this->CandidateCall, DL)); in onAnalysisStart()
1338 // analyzer - instead, we should abstract it to a common method in the in onAnalysisStart()
1340 int SingleBBBonusPercent = 50; in onAnalysisStart()
1341 int VectorBonusPercent = TTI.getInlinerVectorBonusPercent(); in onAnalysisStart()
1342 Threshold += TTI.adjustInliningThreshold(&CandidateCall); in onAnalysisStart()
1343 Threshold *= TTI.getInliningThresholdMultiplier(); in onAnalysisStart()
1344 SingleBBBonus = Threshold * SingleBBBonusPercent / 100; in onAnalysisStart()
1345 VectorBonus = Threshold * VectorBonusPercent / 100; in onAnalysisStart()
1346 Threshold += (SingleBBBonus + VectorBonus); in onAnalysisStart()
1365 /// Test whether the given value is an Alloca-derived function argument.
1379 // The threshold delta is printed only when it is non-zero. It happens in emitInstructionAnnot()
1381 std::optional<InstructionCostDetail> Record = ICCA->getCostDetails(I); in emitInstructionAnnot()
1385 OS << "; cost before = " << Record->CostBefore in emitInstructionAnnot()
1386 << ", cost after = " << Record->CostAfter in emitInstructionAnnot()
1387 << ", threshold before = " << Record->ThresholdBefore in emitInstructionAnnot()
1388 << ", threshold after = " << Record->ThresholdAfter << ", "; in emitInstructionAnnot()
1389 OS << "cost delta = " << Record->getCostDelta(); in emitInstructionAnnot()
1390 if (Record->hasThresholdChanged()) in emitInstructionAnnot()
1391 OS << ", threshold delta = " << Record->getThresholdDelta(); in emitInstructionAnnot()
1393 auto C = ICCA->getSimplifiedValue(const_cast<Instruction *>(I)); in emitInstructionAnnot()
1396 (*C)->print(OS, true); in emitInstructionAnnot()
1431 if (OpC->isZero()) in accumulateGEPOffset()
1436 unsigned ElementIdx = OpC->getZExtValue(); in accumulateGEPOffset()
1438 Offset += APInt(IntPtrWidth, SL->getElementOffset(ElementIdx)); in accumulateGEPOffset()
1443 Offset += OpC->getValue().sextOrTrunc(IntPtrWidth) * TypeSize; in accumulateGEPOffset()
1475 // a threshold. in visitAlloca()
1476 // FIXME: If the threshold is removed or lowered too much, we could end up in visitAlloca()
1477 // being too pessimistic and prevent inlining non-problematic code. This in visitAlloca()
1482 AllocSize->getLimitedValue(), in visitAlloca()
1512 // Phi nodes are always zero-cost. in visitPHI()
1519 bool CheckSROA = I.getType()->isPointerTy(); in visitPHI()
1601 /// Check we can fold GEPs of constant-offset call site argument pointers.
1676 /// (CandidateCall->getParent()->getParent()), we might be able to determine
1686 Type *RT = CB.getFunctionType()->getReturnType(); in simplifyIntrinsicCallIsConstant()
1694 if (cast<ConstantInt>(CB.getArgOperand(3))->isOne()) in simplifyIntrinsicCallObjectSize()
1732 unsigned IntegerSize = I.getType()->getScalarSizeInBits(); in visitPtrToInt()
1733 unsigned AS = I.getOperand(0)->getType()->getPointerAddressSpace(); in visitPtrToInt()
1746 // preserved either cannot fire on an integer, or won't in-and-of themselves in visitPtrToInt()
1760 // Track base/offset pairs when round-tripped through a pointer without in visitIntToPtr()
1763 unsigned IntegerSize = Op->getType()->getScalarSizeInBits(); in visitIntToPtr()
1787 // If this is a floating-point cast, and the target says this operation in visitCastInst()
1809 return CandidateCall.paramHasAttr(A->getArgNo(), Attr); in paramHasAttr()
1815 // caller. This will also trip if the callee function has a non-null in isKnownNonNullInCallee()
1827 // alloca-derived value and null. Note that this fires regardless of in isKnownNonNullInCallee()
1836 // site is unreachable-terminated, there is little point in inlining this in allowSizeGrowth()
1838 // FIXME: Note that it is possible that an unreachable-terminated block has a in allowSizeGrowth()
1849 // general threshold adjusting heuristics in updateThreshold(). in allowSizeGrowth()
1851 if (isa<UnreachableInst>(II->getNormalDest()->getTerminator())) in allowSizeGrowth()
1853 } else if (isa<UnreachableInst>(Call.getParent()->getTerminator())) in allowSizeGrowth()
1863 if (PSI && PSI->hasProfileSummary()) in isColdCallSite()
1864 return PSI->isColdCallSite(Call, CallerBFI); in isColdCallSite()
1876 auto CallSiteFreq = CallerBFI->getBlockFreq(CallSiteBB); in isColdCallSite()
1878 CallerBFI->getBlockFreq(&(Call.getCaller()->getEntryBlock())); in isColdCallSite()
1882 std::optional<int>
1888 if (PSI && PSI->hasProfileSummary() && PSI->isHotCallSite(Call, CallerBFI)) in getHotCallSiteThreshold()
1892 // threshold. in getHotCallSiteThreshold()
1901 BlockFrequency CallSiteFreq = CallerBFI->getBlockFreq(CallSiteBB); in getHotCallSiteThreshold()
1902 BlockFrequency CallerEntryFreq = CallerBFI->getEntryFreq(); in getHotCallSiteThreshold()
1912 // If no size growth is allowed for this inlining, set Threshold to 0. in updateThreshold()
1914 Threshold = 0; in updateThreshold()
1921 auto MinIfValid = [](int A, std::optional<int> B) { in updateThreshold()
1926 auto MaxIfValid = [](int A, std::optional<int> B) { in updateThreshold()
1930 // Various bonus percentages. These are multiplied by Threshold to get the in updateThreshold()
1942 int SingleBBBonusPercent = 50; in updateThreshold()
1943 int VectorBonusPercent = TTI.getInlinerVectorBonusPercent(); in updateThreshold()
1944 int LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus; in updateThreshold()
1954 // and reduce the threshold if the caller has the necessary attribute. in updateThreshold()
1955 if (Caller->hasMinSize()) { in updateThreshold()
1956 Threshold = MinIfValid(Threshold, Params.OptMinSizeThreshold); in updateThreshold()
1958 // bonuses, but not the last-call-to-static bonus. Inlining the last call to in updateThreshold()
1963 } else if (Caller->hasOptSize()) in updateThreshold()
1964 Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); in updateThreshold()
1966 // Adjust the threshold based on inlinehint attribute and profile based in updateThreshold()
1968 if (!Caller->hasMinSize()) { in updateThreshold()
1970 Threshold = MaxIfValid(Threshold, Params.HintThreshold); in updateThreshold()
1981 if (!Caller->hasOptSize() && HotCallSiteThreshold) { in updateThreshold()
1983 // FIXME: This should update the threshold only if it exceeds the in updateThreshold()
1984 // current threshold, but AutoFDO + ThinLTO currently relies on this in updateThreshold()
1987 Threshold = *HotCallSiteThreshold; in updateThreshold()
1992 // reduction, it can cause the size of a non-cold caller to increase in updateThreshold()
1995 Threshold = MinIfValid(Threshold, Params.ColdCallSiteThreshold); in updateThreshold()
1999 if (PSI->isFunctionEntryHot(&Callee)) { in updateThreshold()
2002 // that the callee is hot and treat it as a weaker hint for threshold in updateThreshold()
2004 Threshold = MaxIfValid(Threshold, Params.HintThreshold); in updateThreshold()
2005 } else if (PSI->isFunctionEntryCold(&Callee)) { in updateThreshold()
2009 // reduction, it can cause the size of a non-cold caller to increase in updateThreshold()
2012 Threshold = MinIfValid(Threshold, Params.ColdThreshold); in updateThreshold()
2017 Threshold += TTI.adjustInliningThreshold(&Call); in updateThreshold()
2019 // Finally, take the target-specific inlining threshold multiplier into in updateThreshold()
2021 Threshold *= TTI.getInliningThresholdMultiplier(); in updateThreshold()
2023 SingleBBBonus = Threshold * SingleBBBonusPercent / 100; in updateThreshold()
2024 VectorBonus = Threshold * VectorBonusPercent / 100; in updateThreshold()
2030 Cost -= LastCallToStaticBonus; in updateThreshold()
2065 if (!Instr->getMetadata(LLVMContext::MD_make_implicit)) in visitCmpInst()
2089 // constant-related pointers. in visitSub()
2099 Constant *CLHS = ConstantInt::get(LHS->getContext(), LHSOffset); in visitSub()
2100 Constant *CRHS = ConstantInt::get(RHS->getContext(), RHSOffset); in visitSub()
2126 FI->getFastMathFlags(), DL); in visitBinaryOperator()
2145 if (I.getType()->isFloatingPointTy() && in visitBinaryOperator()
2241 // Try to re-map the arguments to constants. in simplifyCallSite()
2281 if (!F || F->getFunctionType() != Call.getFunctionType()) { in visitCallBase()
2299 switch (II->getIntrinsicID()) { in visitCallBase()
2324 if (auto *SROAArg = getSROAArgForValueOrNull(II->getOperand(0))) in visitCallBase()
2346 if (!(Call.onlyReadsMemory() || (IsIndirectCall && F->onlyReadsMemory()))) in visitCallBase()
2359 // We model unconditional branches as essentially free -- they really in visitBranchInst()
2370 bool CheckSROA = SI.getType()->isPointerTy(); in visitSelectInst()
2409 Value *SelectedV = CondC->isAllOnesValue() ? TrueVal in visitSelectInst()
2410 : (CondC->isNullValue()) ? FalseVal in visitSelectInst()
2521 // We found something we don't understand or can't handle. Mark any SROA-able in visitInstruction()
2533 /// aborts early if the threshold has been exceeded or an impossible to inline
2547 // Similarly, skip pseudo-probes. in analyzeBlock()
2556 if (isa<ExtractElementInst>(I) || I.getType()->isVectorTy()) in analyzeBlock()
2561 // all of the per-instruction logic. The visit tree returns true if we in analyzeBlock()
2589 ORE->emit([&]() { in analyzeBlock()
2606 ORE->emit([&]() { in analyzeBlock()
2631 if (!V->getType()->isPointerTy()) in stripAndComputeInBoundsConstantOffsets()
2634 unsigned AS = V->getType()->getPointerAddressSpace(); in stripAndComputeInBoundsConstantOffsets()
2644 if (!GEP->isInBounds() || !accumulateGEPOffset(*GEP, Offset)) in stripAndComputeInBoundsConstantOffsets()
2646 V = GEP->getPointerOperand(); in stripAndComputeInBoundsConstantOffsets()
2648 if (GA->isInterposable()) in stripAndComputeInBoundsConstantOffsets()
2650 V = GA->getAliasee(); in stripAndComputeInBoundsConstantOffsets()
2654 assert(V->getType()->isPointerTy() && "Unexpected operand type!"); in stripAndComputeInBoundsConstantOffsets()
2657 Type *IdxPtrTy = DL.getIndexType(V->getType()); in stripAndComputeInBoundsConstantOffsets()
2702 /// viable. It computes the cost and adjusts the threshold based on numerous
2704 /// is below the computed threshold, then inlining was forcibly disabled by
2718 for (User *U : Caller->users()) { in analyze()
2720 if (Call && Call->getFunction() == Caller) { in analyze()
2736 ConstantOffsetPtrs[&FAI] = std::make_pair(PtrArg, C->getValue()); in analyze()
2761 // basic blocks in a breadth-first order as we insert live successors. To in analyze()
2763 // crossing our threshold, we use a small-size optimized SetVector. in analyze()
2774 if (BB->empty()) in analyze()
2785 // invalid cross-function reference. in analyze()
2787 if (BB->hasAddressTaken()) in analyze()
2788 for (User *U : BlockAddress::get(&*BB)->users()) in analyze()
2792 // Analyze the cost of this block. If we blow through the threshold, this in analyze()
2798 Instruction *TI = BB->getTerminator(); in analyze()
2803 if (BI->isConditional()) { in analyze()
2804 Value *Cond = BI->getCondition(); in analyze()
2807 BasicBlock *NextBB = BI->getSuccessor(SimpleCond->isZero() ? 1 : 0); in analyze()
2815 Value *Cond = SI->getCondition(); in analyze()
2818 BasicBlock *NextBB = SI->findCaseValue(SimpleCond)->getCaseSuccessor(); in analyze()
2840 // If the callee's stack size exceeds the user-specified threshold, in analyze()
2845 if (std::optional<int> AttrMaxStackSize = getStringFnAttrAsInt( in analyze()
2870 DEBUG_PRINT_STAT(Threshold); in print()
2896 int llvm::getCallsiteCost(const TargetTransformInfo &TTI, const CallBase &Call, in getCallsiteCost()
2903 PointerType *PTy = cast<PointerType>(Call.getArgOperand(I)->getType()); in getCallsiteCost()
2905 unsigned AS = PTy->getAddressSpace(); in getCallsiteCost()
2908 unsigned NumStores = (TypeSize + PointerSize - 1) / PointerSize; in getCallsiteCost()
2920 // For non-byval arguments subtract off one instruction per call in getCallsiteCost()
2942 std::optional<int> llvm::getInliningCostEstimate( in getInliningCostEstimate()
2989 // before coro-split pass, in getAttributeBasedInliningDecision()
2990 // coro-early pass can not handle this quiet well. in getAttributeBasedInliningDecision()
2992 if (Callee->isPresplitCoroutine()) in getAttributeBasedInliningDecision()
3000 unsigned AllocaAS = Callee->getDataLayout().getAllocaAddrSpace(); in getAttributeBasedInliningDecision()
3003 PointerType *PTy = cast<PointerType>(Call.getArgOperand(I)->getType()); in getAttributeBasedInliningDecision()
3004 if (PTy->getAddressSpace() != AllocaAS) in getAttributeBasedInliningDecision()
3009 // Calls to functions with always-inline attributes should be inlined in getAttributeBasedInliningDecision()
3022 // always-inline attribute). in getAttributeBasedInliningDecision()
3028 if (Caller->hasOptNone()) in getAttributeBasedInliningDecision()
3033 if (!Caller->nullPointerIsDefined() && Callee->nullPointerIsDefined()) in getAttributeBasedInliningDecision()
3036 // Don't inline functions which can be interposed at link-time. in getAttributeBasedInliningDecision()
3037 if (Callee->isInterposable()) in getAttributeBasedInliningDecision()
3041 if (Callee->hasFnAttribute(Attribute::NoInline)) in getAttributeBasedInliningDecision()
3063 if (UserDecision->isSuccess()) in getInlineCost()
3065 return llvm::InlineCost::getNever(UserDecision->getFailureReason()); in getInlineCost()
3068 LLVM_DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() in getInlineCost()
3069 << "... (caller:" << Call.getCaller()->getName() in getInlineCost()
3079 // We use always/never here since threshold is not meaningful, in getInlineCost()
3080 // as it's not what drives cost-benefit analysis. in getInlineCost()
3106 // Disallow inlining of blockaddresses which are used by non-callbr in isInlineViable()
3109 for (User *U : BlockAddress::get(&BB)->users()) in isInlineViable()
3119 Function *Callee = Call->getCalledFunction(); in isInlineViable()
3123 // Disallow calls which expose returns-twice to a function not previously in isInlineViable()
3126 cast<CallInst>(Call)->canReturnTwice()) in isInlineViable()
3127 return InlineResult::failure("exposes returns-twice attribute"); in isInlineViable()
3130 switch (Callee->getIntrinsicID()) { in isInlineViable()
3158 InlineParams llvm::getInlineParams(int Threshold) { in getInlineParams() argument
3161 // This field is the threshold to use for a callee by default. This is in getInlineParams()
3163 // * optimization or size-optimization levels, in getInlineParams()
3165 // * the -inline-threshold flag. in getInlineParams()
3166 // If the -inline-threshold flag is explicitly specified, that is used in getInlineParams()
3171 Params.DefaultThreshold = Threshold; in getInlineParams()
3173 // Set the HintThreshold knob from the -inlinehint-threshold. in getInlineParams()
3176 // Set the HotCallSiteThreshold knob from the -hot-callsite-threshold. in getInlineParams()
3179 // If the -locally-hot-callsite-threshold is explicitly specified, use it to in getInlineParams()
3181 // Params.LocallyHotCallSiteThreshold from -locally-hot-callsite-threshold if in getInlineParams()
3190 // -inline-cold-callsite-threshold. in getInlineParams()
3194 // -inlinehint-threshold commandline option is not explicitly given. If that in getInlineParams()
3197 // If the -inline-threshold is not specified, set the ColdThreshold from the in getInlineParams()
3198 // -inlinecold-threshold even if it is not explicitly passed. If in getInlineParams()
3199 // -inline-threshold is specified, then -inlinecold-threshold needs to be in getInlineParams()
3215 // Compute the default threshold for inlining based on the opt level and the
3217 static int computeThresholdFromOptLevels(unsigned OptLevel, in computeThresholdFromOptLevels()
3221 if (SizeOptLevel == 1) // -Os in computeThresholdFromOptLevels()
3223 if (SizeOptLevel == 2) // -Oz in computeThresholdFromOptLevels()
3231 // At O3, use the value of -locally-hot-callsite-threshold option to populate in getInlineParams()
3244 [&](Function &F) -> AssumptionCache & { in run()
3260 Function *CalledFunction = CI->getCalledFunction(); in run()
3261 if (!CalledFunction || CalledFunction->isDeclaration()) in run()
3267 OS << " Analyzing call of " << CalledFunction->getName() in run()
3268 << "... (caller:" << CI->getCaller()->getName() << ")\n"; in run()