1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and 10 // related types. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Analysis/InlineAdvisor.h" 15 #include "llvm/ADT/Statistic.h" 16 #include "llvm/Analysis/AssumptionCache.h" 17 #include "llvm/Analysis/InlineCost.h" 18 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 19 #include "llvm/Analysis/ProfileSummaryInfo.h" 20 #include "llvm/Analysis/ReplayInlineAdvisor.h" 21 #include "llvm/Analysis/TargetLibraryInfo.h" 22 #include "llvm/Analysis/TargetTransformInfo.h" 23 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h" 24 #include "llvm/IR/DebugInfoMetadata.h" 25 #include "llvm/IR/PassManager.h" 26 #include "llvm/Support/CommandLine.h" 27 #include "llvm/Support/raw_ostream.h" 28 29 using namespace llvm; 30 #define DEBUG_TYPE "inline" 31 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL 32 #define LLVM_HAVE_TF_AOT 33 #endif 34 35 // This weirdly named statistic tracks the number of times that, when attempting 36 // to inline a function A into B, we analyze the callers of B in order to see 37 // if those would be more profitable and blocked inline steps. 38 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed"); 39 40 /// Flag to add inline messages as callsite attributes 'inline-remark'. 41 static cl::opt<bool> 42 InlineRemarkAttribute("inline-remark-attribute", cl::init(false), 43 cl::Hidden, 44 cl::desc("Enable adding inline-remark attribute to" 45 " callsites processed by inliner but decided" 46 " to be not inlined")); 47 48 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false), 49 cl::Hidden, 50 cl::desc("Enable deferred inlining")); 51 52 // An integer used to limit the cost of inline deferral. The default negative 53 // number tells shouldBeDeferred to only take the secondary cost into account. 54 static cl::opt<int> 55 InlineDeferralScale("inline-deferral-scale", 56 cl::desc("Scale to limit the cost of inline deferral"), 57 cl::init(2), cl::Hidden); 58 59 static cl::opt<bool> 60 AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false), 61 cl::desc("If true, annotate inline advisor remarks " 62 "with LTO and pass information.")); 63 64 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats; 65 66 namespace { 67 using namespace llvm::ore; 68 class MandatoryInlineAdvice : public InlineAdvice { 69 public: 70 MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 71 OptimizationRemarkEmitter &ORE, 72 bool IsInliningMandatory) 73 : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {} 74 75 private: 76 void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); } 77 78 void recordInliningImpl() override { 79 if (IsInliningRecommended) 80 emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended, 81 [&](OptimizationRemark &Remark) { 82 Remark << ": always inline attribute"; 83 }); 84 } 85 86 void recordUnsuccessfulInliningImpl(const InlineResult &Result) override { 87 if (IsInliningRecommended) 88 ORE.emit([&]() { 89 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), 90 "NotInlined", DLoc, Block) 91 << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '" 92 << NV("Caller", Caller) 93 << "': " << NV("Reason", Result.getFailureReason()); 94 }); 95 } 96 97 void recordUnattemptedInliningImpl() override { 98 assert(!IsInliningRecommended && "Expected to attempt inlining"); 99 } 100 }; 101 } // namespace 102 103 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl( 104 const InlineResult &Result) { 105 using namespace ore; 106 llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) + 107 "; " + inlineCostStr(*OIC)); 108 ORE.emit([&]() { 109 return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(), 110 "NotInlined", DLoc, Block) 111 << "'" << NV("Callee", Callee) << "' is not inlined into '" 112 << NV("Caller", Caller) 113 << "': " << NV("Reason", Result.getFailureReason()); 114 }); 115 } 116 117 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() { 118 if (EmitRemarks) 119 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, 120 /* ForProfileContext= */ false, 121 Advisor->getAnnotatedInlinePassName()); 122 } 123 124 void DefaultInlineAdvice::recordInliningImpl() { 125 if (EmitRemarks) 126 emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC, 127 /* ForProfileContext= */ false, 128 Advisor->getAnnotatedInlinePassName()); 129 } 130 131 llvm::Optional<llvm::InlineCost> static getDefaultInlineAdvice( 132 CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) { 133 Function &Caller = *CB.getCaller(); 134 ProfileSummaryInfo *PSI = 135 FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller) 136 .getCachedResult<ProfileSummaryAnalysis>( 137 *CB.getParent()->getParent()->getParent()); 138 139 auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller); 140 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 141 return FAM.getResult<AssumptionAnalysis>(F); 142 }; 143 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { 144 return FAM.getResult<BlockFrequencyAnalysis>(F); 145 }; 146 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { 147 return FAM.getResult<TargetLibraryAnalysis>(F); 148 }; 149 150 auto GetInlineCost = [&](CallBase &CB) { 151 Function &Callee = *CB.getCalledFunction(); 152 auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee); 153 bool RemarksEnabled = 154 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled( 155 DEBUG_TYPE); 156 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI, 157 GetBFI, PSI, RemarksEnabled ? &ORE : nullptr); 158 }; 159 return llvm::shouldInline( 160 CB, GetInlineCost, ORE, 161 Params.EnableDeferral.value_or(EnableInlineDeferral)); 162 } 163 164 std::unique_ptr<InlineAdvice> 165 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) { 166 auto OIC = getDefaultInlineAdvice(CB, FAM, Params); 167 return std::make_unique<DefaultInlineAdvice>( 168 this, CB, OIC, 169 FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller())); 170 } 171 172 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB, 173 OptimizationRemarkEmitter &ORE, 174 bool IsInliningRecommended) 175 : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()), 176 DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE), 177 IsInliningRecommended(IsInliningRecommended) {} 178 179 void InlineAdvice::recordInlineStatsIfNeeded() { 180 if (Advisor->ImportedFunctionsStats) 181 Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee); 182 } 183 184 void InlineAdvice::recordInlining() { 185 markRecorded(); 186 recordInlineStatsIfNeeded(); 187 recordInliningImpl(); 188 } 189 190 void InlineAdvice::recordInliningWithCalleeDeleted() { 191 markRecorded(); 192 recordInlineStatsIfNeeded(); 193 recordInliningWithCalleeDeletedImpl(); 194 } 195 196 AnalysisKey InlineAdvisorAnalysis::Key; 197 198 bool InlineAdvisorAnalysis::Result::tryCreate( 199 InlineParams Params, InliningAdvisorMode Mode, 200 const ReplayInlinerSettings &ReplaySettings, InlineContext IC) { 201 auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 202 switch (Mode) { 203 case InliningAdvisorMode::Default: 204 LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n"); 205 Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC)); 206 // Restrict replay to default advisor, ML advisors are stateful so 207 // replay will need augmentations to interleave with them correctly. 208 if (!ReplaySettings.ReplayFile.empty()) { 209 Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(), 210 std::move(Advisor), ReplaySettings, 211 /* EmitRemarks =*/true, IC); 212 } 213 break; 214 case InliningAdvisorMode::Development: 215 #ifdef LLVM_HAVE_TF_API 216 LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n"); 217 Advisor = 218 llvm::getDevelopmentModeAdvisor(M, MAM, [&FAM, Params](CallBase &CB) { 219 auto OIC = getDefaultInlineAdvice(CB, FAM, Params); 220 return OIC.hasValue(); 221 }); 222 #endif 223 break; 224 case InliningAdvisorMode::Release: 225 #ifdef LLVM_HAVE_TF_AOT 226 LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n"); 227 Advisor = llvm::getReleaseModeAdvisor(M, MAM); 228 #endif 229 break; 230 } 231 232 return !!Advisor; 233 } 234 235 /// Return true if inlining of CB can block the caller from being 236 /// inlined which is proved to be more beneficial. \p IC is the 237 /// estimated inline cost associated with callsite \p CB. 238 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the 239 /// caller if \p CB is suppressed for inlining. 240 static bool 241 shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, 242 function_ref<InlineCost(CallBase &CB)> GetInlineCost) { 243 // For now we only handle local or inline functions. 244 if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage()) 245 return false; 246 // If the cost of inlining CB is non-positive, it is not going to prevent the 247 // caller from being inlined into its callers and hence we don't need to 248 // defer. 249 if (IC.getCost() <= 0) 250 return false; 251 // Try to detect the case where the current inlining candidate caller (call 252 // it B) is a static or linkonce-ODR function and is an inlining candidate 253 // elsewhere, and the current candidate callee (call it C) is large enough 254 // that inlining it into B would make B too big to inline later. In these 255 // circumstances it may be best not to inline C into B, but to inline B into 256 // its callers. 257 // 258 // This only applies to static and linkonce-ODR functions because those are 259 // expected to be available for inlining in the translation units where they 260 // are used. Thus we will always have the opportunity to make local inlining 261 // decisions. Importantly the linkonce-ODR linkage covers inline functions 262 // and templates in C++. 263 // 264 // FIXME: All of this logic should be sunk into getInlineCost. It relies on 265 // the internal implementation of the inline cost metrics rather than 266 // treating them as truly abstract units etc. 267 TotalSecondaryCost = 0; 268 // The candidate cost to be imposed upon the current function. 269 int CandidateCost = IC.getCost() - 1; 270 // If the caller has local linkage and can be inlined to all its callers, we 271 // can apply a huge negative bonus to TotalSecondaryCost. 272 bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse(); 273 // This bool tracks what happens if we DO inline C into B. 274 bool InliningPreventsSomeOuterInline = false; 275 unsigned NumCallerUsers = 0; 276 for (User *U : Caller->users()) { 277 CallBase *CS2 = dyn_cast<CallBase>(U); 278 279 // If this isn't a call to Caller (it could be some other sort 280 // of reference) skip it. Such references will prevent the caller 281 // from being removed. 282 if (!CS2 || CS2->getCalledFunction() != Caller) { 283 ApplyLastCallBonus = false; 284 continue; 285 } 286 287 InlineCost IC2 = GetInlineCost(*CS2); 288 ++NumCallerCallersAnalyzed; 289 if (!IC2) { 290 ApplyLastCallBonus = false; 291 continue; 292 } 293 if (IC2.isAlways()) 294 continue; 295 296 // See if inlining of the original callsite would erase the cost delta of 297 // this callsite. We subtract off the penalty for the call instruction, 298 // which we would be deleting. 299 if (IC2.getCostDelta() <= CandidateCost) { 300 InliningPreventsSomeOuterInline = true; 301 TotalSecondaryCost += IC2.getCost(); 302 NumCallerUsers++; 303 } 304 } 305 306 if (!InliningPreventsSomeOuterInline) 307 return false; 308 309 // If all outer calls to Caller would get inlined, the cost for the last 310 // one is set very low by getInlineCost, in anticipation that Caller will 311 // be removed entirely. We did not account for this above unless there 312 // is only one caller of Caller. 313 if (ApplyLastCallBonus) 314 TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus; 315 316 // If InlineDeferralScale is negative, then ignore the cost of primary 317 // inlining -- IC.getCost() multiplied by the number of callers to Caller. 318 if (InlineDeferralScale < 0) 319 return TotalSecondaryCost < IC.getCost(); 320 321 int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers; 322 int Allowance = IC.getCost() * InlineDeferralScale; 323 return TotalCost < Allowance; 324 } 325 326 namespace llvm { 327 static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) { 328 return R << Arg.Val; 329 } 330 331 template <class RemarkT> 332 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) { 333 using namespace ore; 334 if (IC.isAlways()) { 335 R << "(cost=always)"; 336 } else if (IC.isNever()) { 337 R << "(cost=never)"; 338 } else { 339 R << "(cost=" << ore::NV("Cost", IC.getCost()) 340 << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")"; 341 } 342 if (const char *Reason = IC.getReason()) 343 R << ": " << ore::NV("Reason", Reason); 344 return R; 345 } 346 } // namespace llvm 347 348 std::string llvm::inlineCostStr(const InlineCost &IC) { 349 std::string Buffer; 350 raw_string_ostream Remark(Buffer); 351 Remark << IC; 352 return Remark.str(); 353 } 354 355 void llvm::setInlineRemark(CallBase &CB, StringRef Message) { 356 if (!InlineRemarkAttribute) 357 return; 358 359 Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message); 360 CB.addFnAttr(Attr); 361 } 362 363 /// Return the cost only if the inliner should attempt to inline at the given 364 /// CallSite. If we return the cost, we will emit an optimisation remark later 365 /// using that cost, so we won't do so from this function. Return None if 366 /// inlining should not be attempted. 367 Optional<InlineCost> 368 llvm::shouldInline(CallBase &CB, 369 function_ref<InlineCost(CallBase &CB)> GetInlineCost, 370 OptimizationRemarkEmitter &ORE, bool EnableDeferral) { 371 using namespace ore; 372 373 InlineCost IC = GetInlineCost(CB); 374 Instruction *Call = &CB; 375 Function *Callee = CB.getCalledFunction(); 376 Function *Caller = CB.getCaller(); 377 378 if (IC.isAlways()) { 379 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) 380 << ", Call: " << CB << "\n"); 381 return IC; 382 } 383 384 if (!IC) { 385 LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) 386 << ", Call: " << CB << "\n"); 387 if (IC.isNever()) { 388 ORE.emit([&]() { 389 return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call) 390 << "'" << NV("Callee", Callee) << "' not inlined into '" 391 << NV("Caller", Caller) 392 << "' because it should never be inlined " << IC; 393 }); 394 } else { 395 ORE.emit([&]() { 396 return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call) 397 << "'" << NV("Callee", Callee) << "' not inlined into '" 398 << NV("Caller", Caller) << "' because too costly to inline " 399 << IC; 400 }); 401 } 402 setInlineRemark(CB, inlineCostStr(IC)); 403 return None; 404 } 405 406 int TotalSecondaryCost = 0; 407 if (EnableDeferral && 408 shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) { 409 LLVM_DEBUG(dbgs() << " NOT Inlining: " << CB 410 << " Cost = " << IC.getCost() 411 << ", outer Cost = " << TotalSecondaryCost << '\n'); 412 ORE.emit([&]() { 413 return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts", 414 Call) 415 << "Not inlining. Cost of inlining '" << NV("Callee", Callee) 416 << "' increases the cost of inlining '" << NV("Caller", Caller) 417 << "' in other contexts"; 418 }); 419 setInlineRemark(CB, "deferred"); 420 return None; 421 } 422 423 LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CB 424 << '\n'); 425 return IC; 426 } 427 428 std::string llvm::formatCallSiteLocation(DebugLoc DLoc, 429 const CallSiteFormat &Format) { 430 std::string Buffer; 431 raw_string_ostream CallSiteLoc(Buffer); 432 bool First = true; 433 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { 434 if (!First) 435 CallSiteLoc << " @ "; 436 // Note that negative line offset is actually possible, but we use 437 // unsigned int to match line offset representation in remarks so 438 // it's directly consumable by relay advisor. 439 uint32_t Offset = 440 DIL->getLine() - DIL->getScope()->getSubprogram()->getLine(); 441 uint32_t Discriminator = DIL->getBaseDiscriminator(); 442 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); 443 if (Name.empty()) 444 Name = DIL->getScope()->getSubprogram()->getName(); 445 CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset); 446 if (Format.outputColumn()) 447 CallSiteLoc << ":" << llvm::utostr(DIL->getColumn()); 448 if (Format.outputDiscriminator() && Discriminator) 449 CallSiteLoc << "." << llvm::utostr(Discriminator); 450 First = false; 451 } 452 453 return CallSiteLoc.str(); 454 } 455 456 void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) { 457 if (!DLoc) { 458 return; 459 } 460 461 bool First = true; 462 Remark << " at callsite "; 463 for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) { 464 if (!First) 465 Remark << " @ "; 466 unsigned int Offset = DIL->getLine(); 467 Offset -= DIL->getScope()->getSubprogram()->getLine(); 468 unsigned int Discriminator = DIL->getBaseDiscriminator(); 469 StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName(); 470 if (Name.empty()) 471 Name = DIL->getScope()->getSubprogram()->getName(); 472 Remark << Name << ":" << ore::NV("Line", Offset) << ":" 473 << ore::NV("Column", DIL->getColumn()); 474 if (Discriminator) 475 Remark << "." << ore::NV("Disc", Discriminator); 476 First = false; 477 } 478 479 Remark << ";"; 480 } 481 482 void llvm::emitInlinedInto( 483 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, 484 const Function &Callee, const Function &Caller, bool AlwaysInline, 485 function_ref<void(OptimizationRemark &)> ExtraContext, 486 const char *PassName) { 487 ORE.emit([&]() { 488 StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined"; 489 OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName, 490 DLoc, Block); 491 Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '" 492 << ore::NV("Caller", &Caller) << "'"; 493 if (ExtraContext) 494 ExtraContext(Remark); 495 addLocationToRemarks(Remark, DLoc); 496 return Remark; 497 }); 498 } 499 500 void llvm::emitInlinedIntoBasedOnCost( 501 OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block, 502 const Function &Callee, const Function &Caller, const InlineCost &IC, 503 bool ForProfileContext, const char *PassName) { 504 llvm::emitInlinedInto( 505 ORE, DLoc, Block, Callee, Caller, IC.isAlways(), 506 [&](OptimizationRemark &Remark) { 507 if (ForProfileContext) 508 Remark << " to match profiling context"; 509 Remark << " with " << IC; 510 }, 511 PassName); 512 } 513 514 InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM, 515 Optional<InlineContext> IC) 516 : M(M), FAM(FAM), IC(IC), 517 AnnotatedInlinePassName((IC && AnnotateInlinePhase) 518 ? llvm::AnnotateInlinePassName(*IC) 519 : DEBUG_TYPE) { 520 if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) { 521 ImportedFunctionsStats = 522 std::make_unique<ImportedFunctionsInliningStatistics>(); 523 ImportedFunctionsStats->setModuleInfo(M); 524 } 525 } 526 527 InlineAdvisor::~InlineAdvisor() { 528 if (ImportedFunctionsStats) { 529 assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No); 530 ImportedFunctionsStats->dump(InlinerFunctionImportStats == 531 InlinerFunctionImportStatsOpts::Verbose); 532 } 533 } 534 535 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB, 536 bool Advice) { 537 return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB), 538 Advice); 539 } 540 541 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) { 542 switch (LTOPhase) { 543 case (ThinOrFullLTOPhase::None): 544 return "main"; 545 case (ThinOrFullLTOPhase::ThinLTOPreLink): 546 case (ThinOrFullLTOPhase::FullLTOPreLink): 547 return "prelink"; 548 case (ThinOrFullLTOPhase::ThinLTOPostLink): 549 case (ThinOrFullLTOPhase::FullLTOPostLink): 550 return "postlink"; 551 } 552 llvm_unreachable("unreachable"); 553 } 554 555 static inline const char *getInlineAdvisorContext(InlinePass IP) { 556 switch (IP) { 557 case (InlinePass::AlwaysInliner): 558 return "always-inline"; 559 case (InlinePass::CGSCCInliner): 560 return "cgscc-inline"; 561 case (InlinePass::EarlyInliner): 562 return "early-inline"; 563 case (InlinePass::MLInliner): 564 return "ml-inline"; 565 case (InlinePass::ModuleInliner): 566 return "module-inline"; 567 case (InlinePass::ReplayCGSCCInliner): 568 return "replay-cgscc-inline"; 569 case (InlinePass::ReplaySampleProfileInliner): 570 return "replay-sample-profile-inline"; 571 case (InlinePass::SampleProfileInliner): 572 return "sample-profile-inline"; 573 } 574 575 llvm_unreachable("unreachable"); 576 } 577 578 std::string llvm::AnnotateInlinePassName(InlineContext IC) { 579 return std::string(getLTOPhase(IC.LTOPhase)) + "-" + 580 std::string(getInlineAdvisorContext(IC.Pass)); 581 } 582 583 InlineAdvisor::MandatoryInliningKind 584 InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM, 585 OptimizationRemarkEmitter &ORE) { 586 auto &Callee = *CB.getCalledFunction(); 587 588 auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { 589 return FAM.getResult<TargetLibraryAnalysis>(F); 590 }; 591 592 auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee); 593 594 auto TrivialDecision = 595 llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI); 596 597 if (TrivialDecision) { 598 if (TrivialDecision->isSuccess()) 599 return MandatoryInliningKind::Always; 600 else 601 return MandatoryInliningKind::Never; 602 } 603 return MandatoryInliningKind::NotMandatory; 604 } 605 606 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB, 607 bool MandatoryOnly) { 608 if (!MandatoryOnly) 609 return getAdviceImpl(CB); 610 bool Advice = CB.getCaller() != CB.getCalledFunction() && 611 MandatoryInliningKind::Always == 612 getMandatoryKind(CB, FAM, getCallerORE(CB)); 613 return getMandatoryAdvice(CB, Advice); 614 } 615 616 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) { 617 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()); 618 } 619 620 PreservedAnalyses 621 InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) { 622 const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M); 623 if (!IA) 624 OS << "No Inline Advisor\n"; 625 else 626 IA->getAdvisor()->print(OS); 627 return PreservedAnalyses::all(); 628 } 629 630 PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run( 631 LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG, 632 CGSCCUpdateResult &UR) { 633 const auto &MAMProxy = 634 AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG); 635 636 if (InitialC.size() == 0) { 637 OS << "SCC is empty!\n"; 638 return PreservedAnalyses::all(); 639 } 640 Module &M = *InitialC.begin()->getFunction().getParent(); 641 const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M); 642 if (!IA) 643 OS << "No Inline Advisor\n"; 644 else 645 IA->getAdvisor()->print(OS); 646 return PreservedAnalyses::all(); 647 } 648