1 //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements a custom inliner that handles only functions that 10 // are marked as "always inline". 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/IPO/AlwaysInliner.h" 15 #include "llvm/ADT/SetVector.h" 16 #include "llvm/Analysis/AliasAnalysis.h" 17 #include "llvm/Analysis/AssumptionCache.h" 18 #include "llvm/Analysis/InlineAdvisor.h" 19 #include "llvm/Analysis/InlineCost.h" 20 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 21 #include "llvm/Analysis/ProfileSummaryInfo.h" 22 #include "llvm/IR/Module.h" 23 #include "llvm/InitializePasses.h" 24 #include "llvm/Transforms/Utils/Cloning.h" 25 #include "llvm/Transforms/Utils/ModuleUtils.h" 26 27 using namespace llvm; 28 29 #define DEBUG_TYPE "inline" 30 31 namespace { 32 33 bool AlwaysInlineImpl( 34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI, 35 function_ref<AssumptionCache &(Function &)> GetAssumptionCache, 36 function_ref<AAResults &(Function &)> GetAAR, 37 function_ref<BlockFrequencyInfo &(Function &)> GetBFI) { 38 SmallSetVector<CallBase *, 16> Calls; 39 bool Changed = false; 40 SmallVector<Function *, 16> InlinedComdatFunctions; 41 42 for (Function &F : make_early_inc_range(M)) { 43 if (F.isPresplitCoroutine()) 44 continue; 45 46 if (F.isDeclaration() || !isInlineViable(F).isSuccess()) 47 continue; 48 49 Calls.clear(); 50 51 for (User *U : F.users()) 52 if (auto *CB = dyn_cast<CallBase>(U)) 53 if (CB->getCalledFunction() == &F && 54 CB->hasFnAttr(Attribute::AlwaysInline) && 55 !CB->getAttributes().hasFnAttr(Attribute::NoInline)) 56 Calls.insert(CB); 57 58 for (CallBase *CB : Calls) { 59 Function *Caller = CB->getCaller(); 60 OptimizationRemarkEmitter ORE(Caller); 61 DebugLoc DLoc = CB->getDebugLoc(); 62 BasicBlock *Block = CB->getParent(); 63 64 InlineFunctionInfo IFI(GetAssumptionCache, &PSI, 65 GetBFI ? &GetBFI(*Caller) : nullptr, 66 GetBFI ? &GetBFI(F) : nullptr); 67 68 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true, 69 &GetAAR(F), InsertLifetime); 70 if (!Res.isSuccess()) { 71 ORE.emit([&]() { 72 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) 73 << "'" << ore::NV("Callee", &F) << "' is not inlined into '" 74 << ore::NV("Caller", Caller) 75 << "': " << ore::NV("Reason", Res.getFailureReason()); 76 }); 77 continue; 78 } 79 80 emitInlinedIntoBasedOnCost( 81 ORE, DLoc, Block, F, *Caller, 82 InlineCost::getAlways("always inline attribute"), 83 /*ForProfileContext=*/false, DEBUG_TYPE); 84 85 Changed = true; 86 } 87 88 F.removeDeadConstantUsers(); 89 if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) { 90 // Remember to try and delete this function afterward. This allows to call 91 // filterDeadComdatFunctions() only once. 92 if (F.hasComdat()) { 93 InlinedComdatFunctions.push_back(&F); 94 } else { 95 M.getFunctionList().erase(F); 96 Changed = true; 97 } 98 } 99 } 100 101 if (!InlinedComdatFunctions.empty()) { 102 // Now we just have the comdat functions. Filter out the ones whose comdats 103 // are not actually dead. 104 filterDeadComdatFunctions(InlinedComdatFunctions); 105 // The remaining functions are actually dead. 106 for (Function *F : InlinedComdatFunctions) { 107 M.getFunctionList().erase(F); 108 Changed = true; 109 } 110 } 111 112 return Changed; 113 } 114 115 struct AlwaysInlinerLegacyPass : public ModulePass { 116 bool InsertLifetime; 117 118 AlwaysInlinerLegacyPass() 119 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {} 120 121 AlwaysInlinerLegacyPass(bool InsertLifetime) 122 : ModulePass(ID), InsertLifetime(InsertLifetime) { 123 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 124 } 125 126 /// Main run interface method. We override here to avoid calling skipSCC(). 127 bool runOnModule(Module &M) override { 128 129 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 130 auto GetAAR = [&](Function &F) -> AAResults & { 131 return getAnalysis<AAResultsWrapperPass>(F).getAAResults(); 132 }; 133 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 134 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 135 }; 136 137 return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR, 138 /*GetBFI*/ nullptr); 139 } 140 141 static char ID; // Pass identification, replacement for typeid 142 143 void getAnalysisUsage(AnalysisUsage &AU) const override { 144 AU.addRequired<AssumptionCacheTracker>(); 145 AU.addRequired<AAResultsWrapperPass>(); 146 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 147 } 148 }; 149 150 } // namespace 151 152 char AlwaysInlinerLegacyPass::ID = 0; 153 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", 154 "Inliner for always_inline functions", false, false) 155 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 156 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 157 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 158 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", 159 "Inliner for always_inline functions", false, false) 160 161 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { 162 return new AlwaysInlinerLegacyPass(InsertLifetime); 163 } 164 165 PreservedAnalyses AlwaysInlinerPass::run(Module &M, 166 ModuleAnalysisManager &MAM) { 167 FunctionAnalysisManager &FAM = 168 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 169 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 170 return FAM.getResult<AssumptionAnalysis>(F); 171 }; 172 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { 173 return FAM.getResult<BlockFrequencyAnalysis>(F); 174 }; 175 auto GetAAR = [&](Function &F) -> AAResults & { 176 return FAM.getResult<AAManager>(F); 177 }; 178 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M); 179 180 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, 181 GetAAR, GetBFI); 182 183 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 184 } 185