1 //===- InlineCost.cpp - Cost analysis for inliner -------------------------===//
9 // This file implements inline cost analysis.
52 #define DEBUG_TYPE "inline-cost"
68              "during inline cost calculation"));
72     cl::desc("Prints comments for instruction based on inline cost analysis"));
88     "inline-enable-cost-benefit-analysis", cl::Hidden, cl::init(false),
89     cl::desc("Enable the cost-benefit analysis for the inliner"));
104              "savings won't justify the cost"));
139     InstrCost("inline-instr-cost", cl::Hidden, cl::init(5),
140               cl::desc("Cost of a single instruction when inlining"));
143     MemAccessCost("inline-memaccess-cost", cl::Hidden, cl::init(0),
144                   cl::desc("Cost of load/store instruction when inlining"));
163     "inline-cost-full", cl::Hidden,
164     cl::desc("Compute the full inline cost of a call site even when the cost "
204 // This struct is used to store information about inline cost of a
265   /// analysis in the caller function; we want the inline cost query to be
318   /// Called to account for the cost of argument setup for the Call in the
385   /// can cause dramatic shifts in the cost of inlining a function.
512   // Keep a bunch of stats about the cost savings found so we can print them
557   // This DenseMap stores the delta change in cost and threshold after
562   /// Upper bound for the inlining cost. Bonuses are being applied to account
569   /// Attempt to evaluate indirect calls to boost its inline cost.
575   // True if the cost-benefit-analysis-based inliner is enabled.
578   /// Inlining cost measured in abstract units, accounts for all the
582   int Cost = 0;  member in __anon463f3bdb0111::InlineCostCallAnalyzer
584   // The cumulative cost at the beginning of the basic block being analyzed.  At
585   // the end of analyzing each basic block, "Cost - CostAtBBStart" represents
593   // Whether inlining is decided by cost-threshold analysis.
596   // Whether inlining is decided by cost-benefit analysis.
599   // The cost-benefit pair computed by cost-benefit analysis.
607   /// The mapping of caller Alloca values to their accumulated cost savings. If
609   /// cost must be added.
624   /// Handle a capped 'int' increment for Cost.
627     Cost = std::clamp<int64_t>(Inc + Cost, INT_MIN, INT_MAX);  in addCost()
651             getStringFnAttrAsInt(Call, "call-inline-cost")) {  in onCallBaseVisitStart()
654       // inline cost, not just add to it.  in onCallBaseVisitStart()
692         // We were able to inline the indirect call! Subtract the cost from the  in onLoweredCall()
694         Cost -= std::max(0, CA.getThreshold() - CA.getCost());  in onLoweredCall()
697       // Otherwise simply add the cost for merely making the call.  in onLoweredCall()
704     // If suitable for a jump table, consider the cost for the table size and  in onFinalizeSwitch()
706     // Maximum valid cost increased in this function.  in onFinalizeSwitch()
746            "expected this argument to have a cost");  in onAggregateSROAUse()
751   void onBlockStart(const BasicBlock *BB) override { CostAtBBStart = Cost; }  in onBlockStart()
762         ColdSize += Cost - CostAtBBStart;  in onBlockAnalyzed()
778     // This function is called to store the initial cost of inlining before  in onInstructionAnalysisStart()
782     InstructionCostDetailMap[I].CostBefore = Cost;  in onInstructionAnalysisStart()
787     // This function is called to find new values of cost and threshold after  in onInstructionAnalysisFinish()
791     InstructionCostDetailMap[I].CostAfter = Cost;  in onInstructionAnalysisFinish()
857             CandidateCall, "inline-runtime-cost-for-test")) {  in OverrideCycleSavingsAndSizeForTesting()
863   // both the size cost and the cycle savings.  Return std::nullopt if we don't
871     // falling back to the cost-based metric.  in costBenefitAnalysis()
882     // avoid.  Savings come from the call site cost, such as argument setup and  in costBenefitAnalysis()
931     // Remove the cost of the cold basic blocks to model the runtime cost more  in costBenefitAnalysis()
934     int Size = Cost - ColdSize;  in costBenefitAnalysis()
945     // somewhere in the middle, we fall back to the cost-based analysis.  in costBenefitAnalysis()
960     // Otherwise, we fall back to the cost-based analysis.  in costBenefitAnalysis()
978     // Otherwise, fall back to the cost-based analysis.  in costBenefitAnalysis()
1004     // comparing against Cost.  in finalizeAnalysis()
1011             getStringFnAttrAsInt(CandidateCall, "function-inline-cost"))  in finalizeAnalysis()
1012       Cost = *AttrCost;  in finalizeAnalysis()
1017       Cost *= *AttrCostMult;  in finalizeAnalysis()
1028         return InlineResult::failure("Cost over threshold.");  in finalizeAnalysis()
1035     return Cost < std::max(1, Threshold)  in finalizeAnalysis()
1037                : InlineResult::failure("Cost over threshold.");  in finalizeAnalysis()
1044     // the cost, but only when undercounting doesn't matter.  in shouldStop()
1045     if (Cost < Threshold)  in shouldStop()
1056     // Perform some tweaks to the cost and threshold based on the direct  in onAnalysisStart()
1079     // Speculatively apply all possible bonuses to Threshold. If cost exceeds  in onAnalysisStart()
1080     // this Threshold any time, and cost cannot decrease, we can stop processing  in onAnalysisStart()
1091       Cost += InlineConstants::ColdccPenalty;  in onAnalysisStart()
1093     LLVM_DEBUG(dbgs() << "      Initial cost: " << Cost << "\n");  in onAnalysisStart()
1096     if (Cost >= Threshold && !ComputeFullInlineCost)  in onAnalysisStart()
1097       return InlineResult::failure("high cost");  in onAnalysisStart()
1139   int getCost() const { return Cost; }  in getCost()
1155   InlineCostFeatures Cost = {};  member in __anon463f3bdb0111::InlineCostFeaturesAnalyzer
1157   // FIXME: These constants are taken from the heuristic-based cost visitor.
1165   // FIXME: These are taken from the heuristic-based cost visitor: we should
1175     Cost[static_cast<size_t>(Feature)] += Delta;  in increment()
1179     Cost[static_cast<size_t>(Feature)] = Value;  in set()
1337     // FIXME: we shouldn't repeat this logic in both the Features and Cost  in onAnalysisStart()
1360   const InlineCostFeatures &features() const { return Cost; }  in features()
1378   // The cost of inlining of the given instruction is printed always.  in emitInstructionAnnot()
1385     OS << "; cost before = " << Record->CostBefore  in emitInstructionAnnot()
1386        << ", cost after = " << Record->CostAfter  in emitInstructionAnnot()
1389     OS << "cost delta = " << Record->getCostDelta();  in emitInstructionAnnot()
1512   // Phi nodes are always zero-cost.  in visitPHI()
1721   // Bitcasts are always zero cost.  in visitBitCast()
1788   // is expensive, this may eventually become a library call. Treat the cost  in visitCastInst()
1837   // unless there is literally zero cost.  in allowSizeGrowth()
2027   // the cost of inlining it drops dramatically. It may seem odd to update  in updateThreshold()
2028   // Cost in updateThreshold, but the bonus depends on the logic in this method.  in updateThreshold()
2030     Cost -= LastCallToStaticBonus;  in updateThreshold()
2080     // should be treated as simplified and free of cost.  in visitCmpInst()
2142   // is expensive, this may eventually become a library call. Treat the cost  in visitBinaryOperator()
2199   // from the cost even when we finally see a store. We just need to disable  in visitStore()
2459   // of jump table/bit test/binary search tree. The cost of the switch is  in visitSwitchInst()
2491   // the inline cost of a resume instruction.  in visitResumeInst()
2497   // the inline cost of a cleanupret instruction.  in visitCleanupReturnInst()
2503   // the inline cost of a catchret instruction.  in visitCatchReturnInst()
2508   // FIXME: It might be reasonably to discount the cost of instructions leading  in visitUnreachableInst()
2529 /// Analyze a basic block for its contribution to the inline cost.
2532 /// block and accounts for their cost during inlining at this callsite. It
2545     // instructions shouldn't factor into the cost computation, but until then,  in analyzeBlock()
2559     // If the instruction simplified to a constant, there is no cost to this  in analyzeBlock()
2563     // cost should count against inlining.  in analyzeBlock()
2594                  << ") and cost is not fully computed";  in analyzeBlock()
2611                  << ". Cost is not fully computed";  in analyzeBlock()
2702 /// viable. It computes the cost and adjusts the threshold based on numerous
2703 /// factors and heuristics. If this method returns false but the computed cost
2759   // particular call site in order to get more accurate cost estimates. This  in analyze()
2792     // Analyze the cost of this block. If we blow through the threshold, this  in analyze()
2869   DEBUG_PRINT_STAT(Cost);  in print()
2898   int64_t Cost = 0;  in getCallsiteCost()  local
2918       Cost += 2 * NumStores * InstrCost;  in getCallsiteCost()
2922       Cost += InstrCost;  in getCallsiteCost()
2926   Cost += InstrCost;  in getCallsiteCost()
2927   Cost += TTI.getInlineCallPenalty(Call.getCaller(), Call, CallPenalty);  in getCallsiteCost()
2929   return std::min<int64_t>(Cost, INT_MAX);  in getCallsiteCost()
3078   // Always make cost benefit based decision explicit.  in getInlineCost()
3080   // as it's not what drives cost-benefit analysis.  in getInlineCost()
3083       return InlineCost::getAlways("benefit over cost",  in getInlineCost()
3086       return InlineCost::getNever("cost over benefit", CA.getCostBenefitPair());  in getInlineCost()