1 //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements a custom inliner that handles only functions that 10 // are marked as "always inline". 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/IPO/AlwaysInliner.h" 15 #include "llvm/ADT/SetVector.h" 16 #include "llvm/Analysis/AliasAnalysis.h" 17 #include "llvm/Analysis/AssumptionCache.h" 18 #include "llvm/Analysis/InlineCost.h" 19 #include "llvm/Analysis/OptimizationRemarkEmitter.h" 20 #include "llvm/Analysis/ProfileSummaryInfo.h" 21 #include "llvm/IR/Module.h" 22 #include "llvm/InitializePasses.h" 23 #include "llvm/Transforms/IPO/Inliner.h" 24 #include "llvm/Transforms/Utils/Cloning.h" 25 #include "llvm/Transforms/Utils/ModuleUtils.h" 26 27 using namespace llvm; 28 29 #define DEBUG_TYPE "inline" 30 31 namespace { 32 33 bool AlwaysInlineImpl( 34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI, 35 function_ref<AssumptionCache &(Function &)> GetAssumptionCache, 36 function_ref<AAResults &(Function &)> GetAAR, 37 function_ref<BlockFrequencyInfo &(Function &)> GetBFI) { 38 SmallSetVector<CallBase *, 16> Calls; 39 bool Changed = false; 40 SmallVector<Function *, 16> InlinedFunctions; 41 for (Function &F : M) { 42 // When callee coroutine function is inlined into caller coroutine function 43 // before coro-split pass, 44 // coro-early pass can not handle this quiet well. 45 // So we won't inline the coroutine function if it have not been unsplited 46 if (F.isPresplitCoroutine()) 47 continue; 48 49 if (!F.isDeclaration() && isInlineViable(F).isSuccess()) { 50 Calls.clear(); 51 52 for (User *U : F.users()) 53 if (auto *CB = dyn_cast<CallBase>(U)) 54 if (CB->getCalledFunction() == &F && 55 CB->hasFnAttr(Attribute::AlwaysInline) && 56 !CB->getAttributes().hasFnAttr(Attribute::NoInline)) 57 Calls.insert(CB); 58 59 for (CallBase *CB : Calls) { 60 Function *Caller = CB->getCaller(); 61 OptimizationRemarkEmitter ORE(Caller); 62 DebugLoc DLoc = CB->getDebugLoc(); 63 BasicBlock *Block = CB->getParent(); 64 65 InlineFunctionInfo IFI(GetAssumptionCache, &PSI, 66 GetBFI ? &GetBFI(*Caller) : nullptr, 67 GetBFI ? &GetBFI(F) : nullptr); 68 69 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true, 70 &GetAAR(F), InsertLifetime); 71 if (!Res.isSuccess()) { 72 ORE.emit([&]() { 73 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, 74 Block) 75 << "'" << ore::NV("Callee", &F) << "' is not inlined into '" 76 << ore::NV("Caller", Caller) 77 << "': " << ore::NV("Reason", Res.getFailureReason()); 78 }); 79 continue; 80 } 81 82 emitInlinedIntoBasedOnCost( 83 ORE, DLoc, Block, F, *Caller, 84 InlineCost::getAlways("always inline attribute"), 85 /*ForProfileContext=*/false, DEBUG_TYPE); 86 87 Changed = true; 88 } 89 90 if (F.hasFnAttribute(Attribute::AlwaysInline)) { 91 // Remember to try and delete this function afterward. This both avoids 92 // re-walking the rest of the module and avoids dealing with any 93 // iterator invalidation issues while deleting functions. 94 InlinedFunctions.push_back(&F); 95 } 96 } 97 } 98 99 // Remove any live functions. 100 erase_if(InlinedFunctions, [&](Function *F) { 101 F->removeDeadConstantUsers(); 102 return !F->isDefTriviallyDead(); 103 }); 104 105 // Delete the non-comdat ones from the module and also from our vector. 106 auto NonComdatBegin = partition( 107 InlinedFunctions, [&](Function *F) { return F->hasComdat(); }); 108 for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end())) { 109 M.getFunctionList().erase(F); 110 Changed = true; 111 } 112 InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end()); 113 114 if (!InlinedFunctions.empty()) { 115 // Now we just have the comdat functions. Filter out the ones whose comdats 116 // are not actually dead. 117 filterDeadComdatFunctions(InlinedFunctions); 118 // The remaining functions are actually dead. 119 for (Function *F : InlinedFunctions) { 120 M.getFunctionList().erase(F); 121 Changed = true; 122 } 123 } 124 125 return Changed; 126 } 127 128 struct AlwaysInlinerLegacyPass : public ModulePass { 129 bool InsertLifetime; 130 131 AlwaysInlinerLegacyPass() 132 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {} 133 134 AlwaysInlinerLegacyPass(bool InsertLifetime) 135 : ModulePass(ID), InsertLifetime(InsertLifetime) { 136 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 137 } 138 139 /// Main run interface method. We override here to avoid calling skipSCC(). 140 bool runOnModule(Module &M) override { 141 142 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(); 143 auto GetAAR = [&](Function &F) -> AAResults & { 144 return getAnalysis<AAResultsWrapperPass>(F).getAAResults(); 145 }; 146 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 147 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); 148 }; 149 150 return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR, 151 /*GetBFI*/ nullptr); 152 } 153 154 static char ID; // Pass identification, replacement for typeid 155 156 void getAnalysisUsage(AnalysisUsage &AU) const override { 157 AU.addRequired<AssumptionCacheTracker>(); 158 AU.addRequired<AAResultsWrapperPass>(); 159 AU.addRequired<ProfileSummaryInfoWrapperPass>(); 160 } 161 }; 162 163 } // namespace 164 165 char AlwaysInlinerLegacyPass::ID = 0; 166 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", 167 "Inliner for always_inline functions", false, false) 168 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) 169 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 170 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 171 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", 172 "Inliner for always_inline functions", false, false) 173 174 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { 175 return new AlwaysInlinerLegacyPass(InsertLifetime); 176 } 177 178 PreservedAnalyses AlwaysInlinerPass::run(Module &M, 179 ModuleAnalysisManager &MAM) { 180 FunctionAnalysisManager &FAM = 181 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 182 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { 183 return FAM.getResult<AssumptionAnalysis>(F); 184 }; 185 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & { 186 return FAM.getResult<BlockFrequencyAnalysis>(F); 187 }; 188 auto GetAAR = [&](Function &F) -> AAResults & { 189 return FAM.getResult<AAManager>(F); 190 }; 191 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M); 192 193 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, 194 GetAAR, GetBFI); 195 196 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 197 } 198