1 //===- SplitModule.cpp - Split a module into partitions -------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the function llvm::SplitModule, which splits a module
10 // into multiple linkable partitions. It can be used to implement parallel code
11 // generation for link-time optimization.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/Transforms/Utils/SplitModule.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/EquivalenceClasses.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/ADT/SmallVector.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/IR/Comdat.h"
22 #include "llvm/IR/Constant.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/Function.h"
25 #include "llvm/IR/GlobalAlias.h"
26 #include "llvm/IR/GlobalObject.h"
27 #include "llvm/IR/GlobalValue.h"
28 #include "llvm/IR/GlobalVariable.h"
29 #include "llvm/IR/Instruction.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/User.h"
32 #include "llvm/IR/Value.h"
33 #include "llvm/Support/Casting.h"
34 #include "llvm/Support/Debug.h"
35 #include "llvm/Support/ErrorHandling.h"
36 #include "llvm/Support/MD5.h"
37 #include "llvm/Support/raw_ostream.h"
38 #include "llvm/Transforms/Utils/Cloning.h"
39 #include "llvm/Transforms/Utils/ValueMapper.h"
40 #include <algorithm>
41 #include <cassert>
42 #include <iterator>
43 #include <memory>
44 #include <queue>
45 #include <utility>
46 #include <vector>
47
48 using namespace llvm;
49
50 #define DEBUG_TYPE "split-module"
51
52 namespace {
53
54 using ClusterMapType = EquivalenceClasses<const GlobalValue *>;
55 using ComdatMembersType = DenseMap<const Comdat *, const GlobalValue *>;
56 using ClusterIDMapType = DenseMap<const GlobalValue *, unsigned>;
57
compareClusters(const std::pair<unsigned,unsigned> & A,const std::pair<unsigned,unsigned> & B)58 bool compareClusters(const std::pair<unsigned, unsigned> &A,
59 const std::pair<unsigned, unsigned> &B) {
60 if (A.second || B.second)
61 return A.second > B.second;
62 return A.first > B.first;
63 }
64
65 using BalancingQueueType =
66 std::priority_queue<std::pair<unsigned, unsigned>,
67 std::vector<std::pair<unsigned, unsigned>>,
68 decltype(compareClusters) *>;
69
70 } // end anonymous namespace
71
addNonConstUser(ClusterMapType & GVtoClusterMap,const GlobalValue * GV,const User * U)72 static void addNonConstUser(ClusterMapType &GVtoClusterMap,
73 const GlobalValue *GV, const User *U) {
74 assert((!isa<Constant>(U) || isa<GlobalValue>(U)) && "Bad user");
75
76 if (const Instruction *I = dyn_cast<Instruction>(U)) {
77 const GlobalValue *F = I->getParent()->getParent();
78 GVtoClusterMap.unionSets(GV, F);
79 } else if (const GlobalValue *GVU = dyn_cast<GlobalValue>(U)) {
80 GVtoClusterMap.unionSets(GV, GVU);
81 } else {
82 llvm_unreachable("Underimplemented use case");
83 }
84 }
85
86 // Adds all GlobalValue users of V to the same cluster as GV.
addAllGlobalValueUsers(ClusterMapType & GVtoClusterMap,const GlobalValue * GV,const Value * V)87 static void addAllGlobalValueUsers(ClusterMapType &GVtoClusterMap,
88 const GlobalValue *GV, const Value *V) {
89 for (const auto *U : V->users()) {
90 SmallVector<const User *, 4> Worklist;
91 Worklist.push_back(U);
92 while (!Worklist.empty()) {
93 const User *UU = Worklist.pop_back_val();
94 // For each constant that is not a GV (a pure const) recurse.
95 if (isa<Constant>(UU) && !isa<GlobalValue>(UU)) {
96 Worklist.append(UU->user_begin(), UU->user_end());
97 continue;
98 }
99 addNonConstUser(GVtoClusterMap, GV, UU);
100 }
101 }
102 }
103
getGVPartitioningRoot(const GlobalValue * GV)104 static const GlobalObject *getGVPartitioningRoot(const GlobalValue *GV) {
105 const GlobalObject *GO = GV->getAliaseeObject();
106 if (const auto *GI = dyn_cast_or_null<GlobalIFunc>(GO))
107 GO = GI->getResolverFunction();
108 return GO;
109 }
110
111 // Find partitions for module in the way that no locals need to be
112 // globalized.
113 // Try to balance pack those partitions into N files since this roughly equals
114 // thread balancing for the backend codegen step.
findPartitions(Module & M,ClusterIDMapType & ClusterIDMap,unsigned N)115 static void findPartitions(Module &M, ClusterIDMapType &ClusterIDMap,
116 unsigned N) {
117 // At this point module should have the proper mix of globals and locals.
118 // As we attempt to partition this module, we must not change any
119 // locals to globals.
120 LLVM_DEBUG(dbgs() << "Partition module with (" << M.size()
121 << ") functions\n");
122 ClusterMapType GVtoClusterMap;
123 ComdatMembersType ComdatMembers;
124
125 auto recordGVSet = [&GVtoClusterMap, &ComdatMembers](GlobalValue &GV) {
126 if (GV.isDeclaration())
127 return;
128
129 if (!GV.hasName())
130 GV.setName("__llvmsplit_unnamed");
131
132 // Comdat groups must not be partitioned. For comdat groups that contain
133 // locals, record all their members here so we can keep them together.
134 // Comdat groups that only contain external globals are already handled by
135 // the MD5-based partitioning.
136 if (const Comdat *C = GV.getComdat()) {
137 auto &Member = ComdatMembers[C];
138 if (Member)
139 GVtoClusterMap.unionSets(Member, &GV);
140 else
141 Member = &GV;
142 }
143
144 // Aliases should not be separated from their aliasees and ifuncs should
145 // not be separated from their resolvers regardless of linkage.
146 if (const GlobalObject *Root = getGVPartitioningRoot(&GV))
147 if (&GV != Root)
148 GVtoClusterMap.unionSets(&GV, Root);
149
150 if (const Function *F = dyn_cast<Function>(&GV)) {
151 for (const BasicBlock &BB : *F) {
152 BlockAddress *BA = BlockAddress::lookup(&BB);
153 if (!BA || !BA->isConstantUsed())
154 continue;
155 addAllGlobalValueUsers(GVtoClusterMap, F, BA);
156 }
157 }
158
159 if (GV.hasLocalLinkage())
160 addAllGlobalValueUsers(GVtoClusterMap, &GV, &GV);
161 };
162
163 llvm::for_each(M.functions(), recordGVSet);
164 llvm::for_each(M.globals(), recordGVSet);
165 llvm::for_each(M.aliases(), recordGVSet);
166
167 // Assigned all GVs to merged clusters while balancing number of objects in
168 // each.
169 BalancingQueueType BalancingQueue(compareClusters);
170 // Pre-populate priority queue with N slot blanks.
171 for (unsigned i = 0; i < N; ++i)
172 BalancingQueue.push(std::make_pair(i, 0));
173
174 using SortType = std::pair<unsigned, ClusterMapType::iterator>;
175
176 SmallVector<SortType, 64> Sets;
177 SmallPtrSet<const GlobalValue *, 32> Visited;
178
179 // To guarantee determinism, we have to sort SCC according to size.
180 // When size is the same, use leader's name.
181 for (ClusterMapType::iterator I = GVtoClusterMap.begin(),
182 E = GVtoClusterMap.end();
183 I != E; ++I)
184 if (I->isLeader())
185 Sets.push_back(
186 std::make_pair(std::distance(GVtoClusterMap.member_begin(I),
187 GVtoClusterMap.member_end()),
188 I));
189
190 llvm::sort(Sets, [](const SortType &a, const SortType &b) {
191 if (a.first == b.first)
192 return a.second->getData()->getName() > b.second->getData()->getName();
193 else
194 return a.first > b.first;
195 });
196
197 for (auto &I : Sets) {
198 unsigned CurrentClusterID = BalancingQueue.top().first;
199 unsigned CurrentClusterSize = BalancingQueue.top().second;
200 BalancingQueue.pop();
201
202 LLVM_DEBUG(dbgs() << "Root[" << CurrentClusterID << "] cluster_size("
203 << I.first << ") ----> " << I.second->getData()->getName()
204 << "\n");
205
206 for (ClusterMapType::member_iterator MI =
207 GVtoClusterMap.findLeader(I.second);
208 MI != GVtoClusterMap.member_end(); ++MI) {
209 if (!Visited.insert(*MI).second)
210 continue;
211 LLVM_DEBUG(dbgs() << "----> " << (*MI)->getName()
212 << ((*MI)->hasLocalLinkage() ? " l " : " e ") << "\n");
213 Visited.insert(*MI);
214 ClusterIDMap[*MI] = CurrentClusterID;
215 CurrentClusterSize++;
216 }
217 // Add this set size to the number of entries in this cluster.
218 BalancingQueue.push(std::make_pair(CurrentClusterID, CurrentClusterSize));
219 }
220 }
221
externalize(GlobalValue * GV)222 static void externalize(GlobalValue *GV) {
223 if (GV->hasLocalLinkage()) {
224 GV->setLinkage(GlobalValue::ExternalLinkage);
225 GV->setVisibility(GlobalValue::HiddenVisibility);
226 }
227
228 // Unnamed entities must be named consistently between modules. setName will
229 // give a distinct name to each such entity.
230 if (!GV->hasName())
231 GV->setName("__llvmsplit_unnamed");
232 }
233
234 // Returns whether GV should be in partition (0-based) I of N.
isInPartition(const GlobalValue * GV,unsigned I,unsigned N)235 static bool isInPartition(const GlobalValue *GV, unsigned I, unsigned N) {
236 if (const GlobalObject *Root = getGVPartitioningRoot(GV))
237 GV = Root;
238
239 StringRef Name;
240 if (const Comdat *C = GV->getComdat())
241 Name = C->getName();
242 else
243 Name = GV->getName();
244
245 // Partition by MD5 hash. We only need a few bits for evenness as the number
246 // of partitions will generally be in the 1-2 figure range; the low 16 bits
247 // are enough.
248 MD5 H;
249 MD5::MD5Result R;
250 H.update(Name);
251 H.final(R);
252 return (R[0] | (R[1] << 8)) % N == I;
253 }
254
SplitModule(Module & M,unsigned N,function_ref<void (std::unique_ptr<Module> MPart)> ModuleCallback,bool PreserveLocals,bool RoundRobin)255 void llvm::SplitModule(
256 Module &M, unsigned N,
257 function_ref<void(std::unique_ptr<Module> MPart)> ModuleCallback,
258 bool PreserveLocals, bool RoundRobin) {
259 if (!PreserveLocals) {
260 for (Function &F : M)
261 externalize(&F);
262 for (GlobalVariable &GV : M.globals())
263 externalize(&GV);
264 for (GlobalAlias &GA : M.aliases())
265 externalize(&GA);
266 for (GlobalIFunc &GIF : M.ifuncs())
267 externalize(&GIF);
268 }
269
270 // This performs splitting without a need for externalization, which might not
271 // always be possible.
272 ClusterIDMapType ClusterIDMap;
273 findPartitions(M, ClusterIDMap, N);
274
275 // Find functions not mapped to modules in ClusterIDMap and count functions
276 // per module. Map unmapped functions using round-robin so that they skip
277 // being distributed by isInPartition() based on function name hashes below.
278 // This provides better uniformity of distribution of functions to modules
279 // in some cases - for example when the number of functions equals to N.
280 if (RoundRobin) {
281 DenseMap<unsigned, unsigned> ModuleFunctionCount;
282 SmallVector<const GlobalValue *> UnmappedFunctions;
283 for (const auto &F : M.functions()) {
284 if (F.isDeclaration() ||
285 F.getLinkage() != GlobalValue::LinkageTypes::ExternalLinkage)
286 continue;
287 auto It = ClusterIDMap.find(&F);
288 if (It == ClusterIDMap.end())
289 UnmappedFunctions.push_back(&F);
290 else
291 ++ModuleFunctionCount[It->second];
292 }
293 BalancingQueueType BalancingQueue(compareClusters);
294 for (unsigned I = 0; I < N; ++I) {
295 if (auto It = ModuleFunctionCount.find(I);
296 It != ModuleFunctionCount.end())
297 BalancingQueue.push(*It);
298 else
299 BalancingQueue.push({I, 0});
300 }
301 for (const auto *const F : UnmappedFunctions) {
302 const unsigned I = BalancingQueue.top().first;
303 const unsigned Count = BalancingQueue.top().second;
304 BalancingQueue.pop();
305 ClusterIDMap.insert({F, I});
306 BalancingQueue.push({I, Count + 1});
307 }
308 }
309
310 // FIXME: We should be able to reuse M as the last partition instead of
311 // cloning it. Note that the callers at the moment expect the module to
312 // be preserved, so will need some adjustments as well.
313 for (unsigned I = 0; I < N; ++I) {
314 ValueToValueMapTy VMap;
315 std::unique_ptr<Module> MPart(
316 CloneModule(M, VMap, [&](const GlobalValue *GV) {
317 if (auto It = ClusterIDMap.find(GV); It != ClusterIDMap.end())
318 return It->second == I;
319 else
320 return isInPartition(GV, I, N);
321 }));
322 if (I != 0)
323 MPart->setModuleInlineAsm("");
324 ModuleCallback(std::move(MPart));
325 }
326 }
327