1 //===- AlwaysInliner.cpp - Code to inline always_inline functions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a custom inliner that handles only functions that
10 // are marked as "always inline".
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/IPO/AlwaysInliner.h"
15 #include "llvm/ADT/SetVector.h"
16 #include "llvm/Analysis/AliasAnalysis.h"
17 #include "llvm/Analysis/AssumptionCache.h"
18 #include "llvm/Analysis/InlineAdvisor.h"
19 #include "llvm/Analysis/InlineCost.h"
20 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
21 #include "llvm/Analysis/ProfileSummaryInfo.h"
22 #include "llvm/IR/Module.h"
23 #include "llvm/InitializePasses.h"
24 #include "llvm/Transforms/Utils/Cloning.h"
25 #include "llvm/Transforms/Utils/ModuleUtils.h"
26
27 using namespace llvm;
28
29 #define DEBUG_TYPE "inline"
30
31 namespace {
32
AlwaysInlineImpl(Module & M,bool InsertLifetime,ProfileSummaryInfo & PSI,function_ref<AssumptionCache & (Function &)> GetAssumptionCache,function_ref<AAResults & (Function &)> GetAAR,function_ref<BlockFrequencyInfo & (Function &)> GetBFI)33 bool AlwaysInlineImpl(
34 Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI,
35 function_ref<AssumptionCache &(Function &)> GetAssumptionCache,
36 function_ref<AAResults &(Function &)> GetAAR,
37 function_ref<BlockFrequencyInfo &(Function &)> GetBFI) {
38 SmallSetVector<CallBase *, 16> Calls;
39 bool Changed = false;
40 SmallVector<Function *, 16> InlinedComdatFunctions;
41
42 for (Function &F : make_early_inc_range(M)) {
43 if (F.isPresplitCoroutine())
44 continue;
45
46 if (F.isDeclaration() || !isInlineViable(F).isSuccess())
47 continue;
48
49 Calls.clear();
50
51 for (User *U : F.users())
52 if (auto *CB = dyn_cast<CallBase>(U))
53 if (CB->getCalledFunction() == &F &&
54 CB->hasFnAttr(Attribute::AlwaysInline) &&
55 !CB->getAttributes().hasFnAttr(Attribute::NoInline))
56 Calls.insert(CB);
57
58 for (CallBase *CB : Calls) {
59 Function *Caller = CB->getCaller();
60 OptimizationRemarkEmitter ORE(Caller);
61 DebugLoc DLoc = CB->getDebugLoc();
62 BasicBlock *Block = CB->getParent();
63
64 InlineFunctionInfo IFI(GetAssumptionCache, &PSI,
65 GetBFI ? &GetBFI(*Caller) : nullptr,
66 GetBFI ? &GetBFI(F) : nullptr);
67
68 InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true,
69 &GetAAR(F), InsertLifetime);
70 if (!Res.isSuccess()) {
71 ORE.emit([&]() {
72 return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block)
73 << "'" << ore::NV("Callee", &F) << "' is not inlined into '"
74 << ore::NV("Caller", Caller)
75 << "': " << ore::NV("Reason", Res.getFailureReason());
76 });
77 continue;
78 }
79
80 emitInlinedIntoBasedOnCost(
81 ORE, DLoc, Block, F, *Caller,
82 InlineCost::getAlways("always inline attribute"),
83 /*ForProfileContext=*/false, DEBUG_TYPE);
84
85 Changed = true;
86 }
87
88 F.removeDeadConstantUsers();
89 if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) {
90 // Remember to try and delete this function afterward. This allows to call
91 // filterDeadComdatFunctions() only once.
92 if (F.hasComdat()) {
93 InlinedComdatFunctions.push_back(&F);
94 } else {
95 M.getFunctionList().erase(F);
96 Changed = true;
97 }
98 }
99 }
100
101 if (!InlinedComdatFunctions.empty()) {
102 // Now we just have the comdat functions. Filter out the ones whose comdats
103 // are not actually dead.
104 filterDeadComdatFunctions(InlinedComdatFunctions);
105 // The remaining functions are actually dead.
106 for (Function *F : InlinedComdatFunctions) {
107 M.getFunctionList().erase(F);
108 Changed = true;
109 }
110 }
111
112 return Changed;
113 }
114
115 struct AlwaysInlinerLegacyPass : public ModulePass {
116 bool InsertLifetime;
117
AlwaysInlinerLegacyPass__anon64ba8fe70111::AlwaysInlinerLegacyPass118 AlwaysInlinerLegacyPass()
119 : AlwaysInlinerLegacyPass(/*InsertLifetime*/ true) {}
120
AlwaysInlinerLegacyPass__anon64ba8fe70111::AlwaysInlinerLegacyPass121 AlwaysInlinerLegacyPass(bool InsertLifetime)
122 : ModulePass(ID), InsertLifetime(InsertLifetime) {
123 initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
124 }
125
126 /// Main run interface method. We override here to avoid calling skipSCC().
runOnModule__anon64ba8fe70111::AlwaysInlinerLegacyPass127 bool runOnModule(Module &M) override {
128
129 auto &PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
130 auto GetAAR = [&](Function &F) -> AAResults & {
131 return getAnalysis<AAResultsWrapperPass>(F).getAAResults();
132 };
133 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
134 return getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
135 };
136
137 return AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache, GetAAR,
138 /*GetBFI*/ nullptr);
139 }
140
141 static char ID; // Pass identification, replacement for typeid
142
getAnalysisUsage__anon64ba8fe70111::AlwaysInlinerLegacyPass143 void getAnalysisUsage(AnalysisUsage &AU) const override {
144 AU.addRequired<AssumptionCacheTracker>();
145 AU.addRequired<AAResultsWrapperPass>();
146 AU.addRequired<ProfileSummaryInfoWrapperPass>();
147 }
148 };
149
150 } // namespace
151
152 char AlwaysInlinerLegacyPass::ID = 0;
153 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
154 "Inliner for always_inline functions", false, false)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)155 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
156 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
157 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
158 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
159 "Inliner for always_inline functions", false, false)
160
161 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
162 return new AlwaysInlinerLegacyPass(InsertLifetime);
163 }
164
run(Module & M,ModuleAnalysisManager & MAM)165 PreservedAnalyses AlwaysInlinerPass::run(Module &M,
166 ModuleAnalysisManager &MAM) {
167 FunctionAnalysisManager &FAM =
168 MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
169 auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
170 return FAM.getResult<AssumptionAnalysis>(F);
171 };
172 auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
173 return FAM.getResult<BlockFrequencyAnalysis>(F);
174 };
175 auto GetAAR = [&](Function &F) -> AAResults & {
176 return FAM.getResult<AAManager>(F);
177 };
178 auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
179
180 bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, GetAssumptionCache,
181 GetAAR, GetBFI);
182
183 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
184 }
185