1 //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a custom inliner that handles only functions that
10 // are marked as "always inline".
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/IPO/AlwaysInliner.h"
15 #include "llvm/ADT/SetVector.h"
16 #include "llvm/Analysis/AliasAnalysis.h"
17 #include "llvm/Analysis/AssumptionCache.h"
18 #include "llvm/Analysis/InlineAdvisor.h"
19 #include "llvm/Analysis/InlineCost.h"
20 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
21 #include "llvm/Analysis/ProfileSummaryInfo.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/InitializePasses.h"
24 #include "llvm/Transforms/Utils/Cloning.h"
25 #include "llvm/Transforms/Utils/ModuleUtils.h"
26
27 using namespace llvm;
28
29 #define DEBUG_TYPE "inline"
30
31 namespace {
32
AlwaysInlineImpl(Module & M,bool InsertLifetime,ProfileSummaryInfo & PSI,FunctionAnalysisManager * FAM,function_ref<AssumptionCache & (Function &)> GetAssumptionCache,function_ref<AAResults & (Function &)> GetAAR)33 bool AlwaysInlineImpl(
34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
35 FunctionAnalysisManager *FAM,
36 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
37 function_ref<AAResults &(Function &)> GetAAR) {
38 SmallSetVector<CallBase *, 16> Calls;
39 bool Changed = false;
40 SmallVector<Function *, 16> InlinedComdatFunctions;
41
42 for (Function &F : make_early_inc_range(M)) {
43 if (F.isPresplitCoroutine())
44 continue;
45
46 if (F.isDeclaration() || !isInlineViable(F).isSuccess())
47 continue;
48
49 Calls.clear();
50
51 for (User *U : F.users())
52 if (auto *CB = dyn_cast<CallBase>(U))
53 if (CB->getCalledFunction() == &F &&
54 CB->hasFnAttr(Attribute::AlwaysInline) &&
55 !CB->getAttributes().hasFnAttr(Attribute::NoInline))
56 Calls.insert(CB);
57
58 for (CallBase *CB : Calls) {
59 Function *Caller = CB->getCaller();
60 OptimizationRemarkEmitter ORE(Caller);
61 DebugLoc DLoc = CB->getDebugLoc();
62 BasicBlock *Block = CB->getParent();
63
64 InlineFunctionInfo IFI(GetAssumptionCache, &PSI, nullptr, nullptr);
65 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
66 &GetAAR(F), InsertLifetime);
67 if (!Res.isSuccess()) {
68 ORE.emit([&]() {
69 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
70 << "'" << ore::NV("Callee", &F) << "' is not inlined into '"
71 << ore::NV("Caller", Caller)
72 << "': " << ore::NV("Reason", Res.getFailureReason());
73 });
74 continue;
75 }
76
77 emitInlinedIntoBasedOnCost(
78 ORE, DLoc, Block, F, *Caller,
79 InlineCost::getAlways("always inline attribute"),
80 /*ForProfileContext=*/false, DEBUG_TYPE);
81
82 Changed = true;
83 if (FAM)
84 FAM->invalidate(*Caller, PreservedAnalyses::none());
85 }
86
87 F.removeDeadConstantUsers();
88 if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
89 // Remember to try and delete this function afterward. This allows to call
90 // filterDeadComdatFunctions() only once.
91 if (F.hasComdat()) {
92 InlinedComdatFunctions.push_back(&F);
93 } else {
94 if (FAM)
95 FAM->clear(F, F.getName());
96 M.getFunctionList().erase(F);
97 Changed = true;
98 }
99 }
100 }
101
102 if (!InlinedComdatFunctions.empty()) {
103 // Now we just have the comdat functions. Filter out the ones whose comdats
104 // are not actually dead.
105 filterDeadComdatFunctions(InlinedComdatFunctions);
106 // The remaining functions are actually dead.
107 for (Function *F : InlinedComdatFunctions) {
108 if (FAM)
109 FAM->clear(*F, F->getName());
110 M.getFunctionList().erase(F);
111 Changed = true;
112 }
113 }
114
115 return Changed;
116 }
117
118 struct AlwaysInlinerLegacyPass : public ModulePass {
119 bool InsertLifetime;
120
AlwaysInlinerLegacyPass__anon64ba8fe70111::AlwaysInlinerLegacyPass121 AlwaysInlinerLegacyPass()
122 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
123
AlwaysInlinerLegacyPass__anon64ba8fe70111::AlwaysInlinerLegacyPass124 AlwaysInlinerLegacyPass(bool InsertLifetime)
125 : ModulePass(ID), InsertLifetime(InsertLifetime) {
126 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
127 }
128
129 /// Main run interface method.
runOnModule__anon64ba8fe70111::AlwaysInlinerLegacyPass130 bool runOnModule(Module &M) override {
131
132 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
133 auto GetAAR = [&](Function &F) -> AAResults & {
134 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
135 };
136 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
137 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
138 };
139
140 return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr,
141 GetAssumptionCache, GetAAR);
142 }
143
144 static char ID; // Pass identification, replacement for typeid
145
getAnalysisUsage__anon64ba8fe70111::AlwaysInlinerLegacyPass146 void getAnalysisUsage(AnalysisUsage &AU) const override {
147 AU.addRequired<AssumptionCacheTracker>();
148 AU.addRequired<AAResultsWrapperPass>();
149 AU.addRequired<ProfileSummaryInfoWrapperPass>();
150 }
151 };
152
153 } // namespace
154
155 char AlwaysInlinerLegacyPass::ID = 0;
156 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
157 "Inliner for always_inline functions", false, false)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)158 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
159 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
160 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
161 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
162 "Inliner for always_inline functions", false, false)
163
164 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
165 return new AlwaysInlinerLegacyPass(InsertLifetime);
166 }
167
run(Module & M,ModuleAnalysisManager & MAM)168 PreservedAnalyses AlwaysInlinerPass::run(Module &M,
169 ModuleAnalysisManager &MAM) {
170 FunctionAnalysisManager &FAM =
171 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
172 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
173 return FAM.getResult<AssumptionAnalysis>(F);
174 };
175 auto GetAAR = [&](Function &F) -> AAResults & {
176 return FAM.getResult<AAManager>(F);
177 };
178 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
179
180 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, &FAM,
181 GetAssumptionCache, GetAAR);
182 if (!Changed)
183 return PreservedAnalyses::all();
184
185 PreservedAnalyses PA;
186 // We have already invalidated all analyses on modified functions.
187 PA.preserveSet<AllAnalysesOn<Function>>();
188 return PA;
189 }
190