1 //===- InlineAlways.cpp - Code to inline always_inline functions ----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements a custom inliner that handles only functions that 10 // are marked as "always inline". 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/IPO/AlwaysInliner.h" 15 #include "llvm/ADT/SetVector.h" 16 #include "llvm/Analysis/AssumptionCache.h" 17 #include "llvm/Analysis/InlineCost.h" 18 #include "llvm/Analysis/TargetLibraryInfo.h" 19 #include "llvm/IR/CallSite.h" 20 #include "llvm/IR/CallingConv.h" 21 #include "llvm/IR/DataLayout.h" 22 #include "llvm/IR/Instructions.h" 23 #include "llvm/IR/Module.h" 24 #include "llvm/IR/Type.h" 25 #include "llvm/Transforms/IPO.h" 26 #include "llvm/Transforms/IPO/Inliner.h" 27 #include "llvm/Transforms/Utils/Cloning.h" 28 #include "llvm/Transforms/Utils/ModuleUtils.h" 29 30 using namespace llvm; 31 32 #define DEBUG_TYPE "inline" 33 34 PreservedAnalyses AlwaysInlinerPass::run(Module &M, 35 ModuleAnalysisManager &MAM) { 36 // Add inline assumptions during code generation. 37 FunctionAnalysisManager &FAM = 38 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager(); 39 std::function<AssumptionCache &(Function &)> GetAssumptionCache = 40 [&](Function &F) -> AssumptionCache & { 41 return FAM.getResult<AssumptionAnalysis>(F); 42 }; 43 InlineFunctionInfo IFI(/*cg=*/nullptr, &GetAssumptionCache); 44 45 SmallSetVector<CallSite, 16> Calls; 46 bool Changed = false; 47 SmallVector<Function *, 16> InlinedFunctions; 48 for (Function &F : M) 49 if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) && 50 isInlineViable(F)) { 51 Calls.clear(); 52 53 for (User *U : F.users()) 54 if (auto CS = CallSite(U)) 55 if (CS.getCalledFunction() == &F) 56 Calls.insert(CS); 57 58 for (CallSite CS : Calls) 59 // FIXME: We really shouldn't be able to fail to inline at this point! 60 // We should do something to log or check the inline failures here. 61 Changed |= 62 InlineFunction(CS, IFI, /*CalleeAAR=*/nullptr, InsertLifetime); 63 64 // Remember to try and delete this function afterward. This both avoids 65 // re-walking the rest of the module and avoids dealing with any iterator 66 // invalidation issues while deleting functions. 67 InlinedFunctions.push_back(&F); 68 } 69 70 // Remove any live functions. 71 erase_if(InlinedFunctions, [&](Function *F) { 72 F->removeDeadConstantUsers(); 73 return !F->isDefTriviallyDead(); 74 }); 75 76 // Delete the non-comdat ones from the module and also from our vector. 77 auto NonComdatBegin = partition( 78 InlinedFunctions, [&](Function *F) { return F->hasComdat(); }); 79 for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end())) 80 M.getFunctionList().erase(F); 81 InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end()); 82 83 if (!InlinedFunctions.empty()) { 84 // Now we just have the comdat functions. Filter out the ones whose comdats 85 // are not actually dead. 86 filterDeadComdatFunctions(M, InlinedFunctions); 87 // The remaining functions are actually dead. 88 for (Function *F : InlinedFunctions) 89 M.getFunctionList().erase(F); 90 } 91 92 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); 93 } 94 95 namespace { 96 97 /// Inliner pass which only handles "always inline" functions. 98 /// 99 /// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner 100 /// base class to provide several facilities such as array alloca merging. 101 class AlwaysInlinerLegacyPass : public LegacyInlinerBase { 102 103 public: 104 AlwaysInlinerLegacyPass() : LegacyInlinerBase(ID, /*InsertLifetime*/ true) { 105 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 106 } 107 108 AlwaysInlinerLegacyPass(bool InsertLifetime) 109 : LegacyInlinerBase(ID, InsertLifetime) { 110 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry()); 111 } 112 113 /// Main run interface method. We override here to avoid calling skipSCC(). 114 bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); } 115 116 static char ID; // Pass identification, replacement for typeid 117 118 InlineCost getInlineCost(CallSite CS) override; 119 120 using llvm::Pass::doFinalization; 121 bool doFinalization(CallGraph &CG) override { 122 return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true); 123 } 124 }; 125 } 126 127 char AlwaysInlinerLegacyPass::ID = 0; 128 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", 129 "Inliner for always_inline functions", false, false) 130 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) 131 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass) 132 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) 133 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) 134 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", 135 "Inliner for always_inline functions", false, false) 136 137 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) { 138 return new AlwaysInlinerLegacyPass(InsertLifetime); 139 } 140 141 /// Get the inline cost for the always-inliner. 142 /// 143 /// The always inliner *only* handles functions which are marked with the 144 /// attribute to force inlining. As such, it is dramatically simpler and avoids 145 /// using the powerful (but expensive) inline cost analysis. Instead it uses 146 /// a very simple and boring direct walk of the instructions looking for 147 /// impossible-to-inline constructs. 148 /// 149 /// Note, it would be possible to go to some lengths to cache the information 150 /// computed here, but as we only expect to do this for relatively few and 151 /// small functions which have the explicit attribute to force inlining, it is 152 /// likely not worth it in practice. 153 InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallSite CS) { 154 Function *Callee = CS.getCalledFunction(); 155 156 // Only inline direct calls to functions with always-inline attributes 157 // that are viable for inlining. 158 if (!Callee) 159 return InlineCost::getNever("indirect call"); 160 161 // FIXME: We shouldn't even get here for declarations. 162 if (Callee->isDeclaration()) 163 return InlineCost::getNever("no definition"); 164 165 if (!CS.hasFnAttr(Attribute::AlwaysInline)) 166 return InlineCost::getNever("no alwaysinline attribute"); 167 168 auto IsViable = isInlineViable(*Callee); 169 if (!IsViable) 170 return InlineCost::getNever(IsViable.message); 171 172 return InlineCost::getAlways("always inliner"); 173 } 174