1 //===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h"
10 #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h"
11 #include "llvm/ExecutionEngine/JITLink/aarch32.h"
12 #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h"
13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
15 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
16 #include "llvm/Support/MemoryBuffer.h"
17 #include <string>
18 #include <vector>
19
20 #define DEBUG_TYPE "orc"
21
22 using namespace llvm;
23 using namespace llvm::jitlink;
24 using namespace llvm::orc;
25
26 namespace {
27
hasInitializerSection(jitlink::LinkGraph & G)28 bool hasInitializerSection(jitlink::LinkGraph &G) {
29 bool IsMachO = G.getTargetTriple().isOSBinFormatMachO();
30 bool IsElf = G.getTargetTriple().isOSBinFormatELF();
31 if (!IsMachO && !IsElf)
32 return false;
33
34 for (auto &Sec : G.sections()) {
35 if (IsMachO && isMachOInitializerSection(Sec.getName()))
36 return true;
37 if (IsElf && isELFInitializerSection(Sec.getName()))
38 return true;
39 }
40
41 return false;
42 }
43
getJITSymbolPtrForSymbol(Symbol & Sym,const Triple & TT)44 ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) {
45 switch (TT.getArch()) {
46 case Triple::arm:
47 case Triple::armeb:
48 case Triple::thumb:
49 case Triple::thumbeb:
50 if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) {
51 // Set LSB to indicate thumb target
52 assert(Sym.isCallable() && "Only callable symbols can have thumb flag");
53 assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear");
54 return Sym.getAddress() + 0x01;
55 }
56 return Sym.getAddress();
57 default:
58 return Sym.getAddress();
59 }
60 }
61
getJITSymbolFlagsForSymbol(Symbol & Sym)62 JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) {
63 JITSymbolFlags Flags;
64
65 if (Sym.getLinkage() == Linkage::Weak)
66 Flags |= JITSymbolFlags::Weak;
67
68 if (Sym.getScope() == Scope::Default)
69 Flags |= JITSymbolFlags::Exported;
70
71 if (Sym.isCallable())
72 Flags |= JITSymbolFlags::Callable;
73
74 return Flags;
75 }
76
77 class LinkGraphMaterializationUnit : public MaterializationUnit {
78 public:
79 static std::unique_ptr<LinkGraphMaterializationUnit>
Create(ObjectLinkingLayer & ObjLinkingLayer,std::unique_ptr<LinkGraph> G)80 Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) {
81 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G);
82 return std::unique_ptr<LinkGraphMaterializationUnit>(
83 new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G),
84 std::move(LGI)));
85 }
86
getName() const87 StringRef getName() const override { return G->getName(); }
materialize(std::unique_ptr<MaterializationResponsibility> MR)88 void materialize(std::unique_ptr<MaterializationResponsibility> MR) override {
89 ObjLinkingLayer.emit(std::move(MR), std::move(G));
90 }
91
92 private:
scanLinkGraph(ExecutionSession & ES,LinkGraph & G)93 static Interface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) {
94
95 Interface LGI;
96
97 auto AddSymbol = [&](Symbol *Sym) {
98 // Skip local symbols.
99 if (Sym->getScope() == Scope::Local)
100 return;
101 assert(Sym->hasName() && "Anonymous non-local symbol?");
102
103 LGI.SymbolFlags[ES.intern(Sym->getName())] =
104 getJITSymbolFlagsForSymbol(*Sym);
105 };
106
107 for (auto *Sym : G.defined_symbols())
108 AddSymbol(Sym);
109 for (auto *Sym : G.absolute_symbols())
110 AddSymbol(Sym);
111
112 if (hasInitializerSection(G))
113 LGI.InitSymbol = makeInitSymbol(ES, G);
114
115 return LGI;
116 }
117
makeInitSymbol(ExecutionSession & ES,LinkGraph & G)118 static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) {
119 std::string InitSymString;
120 raw_string_ostream(InitSymString)
121 << "$." << G.getName() << ".__inits" << Counter++;
122 return ES.intern(InitSymString);
123 }
124
LinkGraphMaterializationUnit(ObjectLinkingLayer & ObjLinkingLayer,std::unique_ptr<LinkGraph> G,Interface LGI)125 LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer,
126 std::unique_ptr<LinkGraph> G, Interface LGI)
127 : MaterializationUnit(std::move(LGI)), ObjLinkingLayer(ObjLinkingLayer),
128 G(std::move(G)) {}
129
discard(const JITDylib & JD,const SymbolStringPtr & Name)130 void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
131 for (auto *Sym : G->defined_symbols())
132 if (Sym->getName() == *Name) {
133 assert(Sym->getLinkage() == Linkage::Weak &&
134 "Discarding non-weak definition");
135 G->makeExternal(*Sym);
136 break;
137 }
138 }
139
140 ObjectLinkingLayer &ObjLinkingLayer;
141 std::unique_ptr<LinkGraph> G;
142 static std::atomic<uint64_t> Counter;
143 };
144
145 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0};
146
147 } // end anonymous namespace
148
149 namespace llvm {
150 namespace orc {
151
152 class ObjectLinkingLayerJITLinkContext final : public JITLinkContext {
153 public:
ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer & Layer,std::unique_ptr<MaterializationResponsibility> MR,std::unique_ptr<MemoryBuffer> ObjBuffer)154 ObjectLinkingLayerJITLinkContext(
155 ObjectLinkingLayer &Layer,
156 std::unique_ptr<MaterializationResponsibility> MR,
157 std::unique_ptr<MemoryBuffer> ObjBuffer)
158 : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer),
159 MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {
160 std::lock_guard<std::mutex> Lock(Layer.LayerMutex);
161 Plugins = Layer.Plugins;
162 }
163
~ObjectLinkingLayerJITLinkContext()164 ~ObjectLinkingLayerJITLinkContext() {
165 // If there is an object buffer return function then use it to
166 // return ownership of the buffer.
167 if (Layer.ReturnObjectBuffer && ObjBuffer)
168 Layer.ReturnObjectBuffer(std::move(ObjBuffer));
169 }
170
getMemoryManager()171 JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
172
notifyMaterializing(LinkGraph & G)173 void notifyMaterializing(LinkGraph &G) {
174 for (auto &P : Plugins)
175 P->notifyMaterializing(*MR, G, *this,
176 ObjBuffer ? ObjBuffer->getMemBufferRef()
177 : MemoryBufferRef());
178 }
179
notifyFailed(Error Err)180 void notifyFailed(Error Err) override {
181 for (auto &P : Plugins)
182 Err = joinErrors(std::move(Err), P->notifyFailed(*MR));
183 Layer.getExecutionSession().reportError(std::move(Err));
184 MR->failMaterialization();
185 }
186
lookup(const LookupMap & Symbols,std::unique_ptr<JITLinkAsyncLookupContinuation> LC)187 void lookup(const LookupMap &Symbols,
188 std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override {
189
190 JITDylibSearchOrder LinkOrder;
191 MR->getTargetJITDylib().withLinkOrderDo(
192 [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; });
193
194 auto &ES = Layer.getExecutionSession();
195
196 SymbolLookupSet LookupSet;
197 for (auto &KV : Symbols) {
198 orc::SymbolLookupFlags LookupFlags;
199 switch (KV.second) {
200 case jitlink::SymbolLookupFlags::RequiredSymbol:
201 LookupFlags = orc::SymbolLookupFlags::RequiredSymbol;
202 break;
203 case jitlink::SymbolLookupFlags::WeaklyReferencedSymbol:
204 LookupFlags = orc::SymbolLookupFlags::WeaklyReferencedSymbol;
205 break;
206 }
207 LookupSet.add(ES.intern(KV.first), LookupFlags);
208 }
209
210 // OnResolve -- De-intern the symbols and pass the result to the linker.
211 auto OnResolve = [LookupContinuation =
212 std::move(LC)](Expected<SymbolMap> Result) mutable {
213 if (!Result)
214 LookupContinuation->run(Result.takeError());
215 else {
216 AsyncLookupResult LR;
217 for (auto &KV : *Result)
218 LR[*KV.first] = KV.second;
219 LookupContinuation->run(std::move(LR));
220 }
221 };
222
223 ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet),
224 SymbolState::Resolved, std::move(OnResolve),
225 [this](const SymbolDependenceMap &Deps) {
226 // Translate LookupDeps map to SymbolSourceJD.
227 for (auto &[DepJD, Deps] : Deps)
228 for (auto &DepSym : Deps)
229 SymbolSourceJDs[NonOwningSymbolStringPtr(DepSym)] = DepJD;
230 });
231 }
232
notifyResolved(LinkGraph & G)233 Error notifyResolved(LinkGraph &G) override {
234 auto &ES = Layer.getExecutionSession();
235
236 SymbolFlagsMap ExtraSymbolsToClaim;
237 bool AutoClaim = Layer.AutoClaimObjectSymbols;
238
239 SymbolMap InternedResult;
240 for (auto *Sym : G.defined_symbols())
241 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
242 auto InternedName = ES.intern(Sym->getName());
243 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple());
244 auto Flags = getJITSymbolFlagsForSymbol(*Sym);
245 InternedResult[InternedName] = {Ptr, Flags};
246 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
247 assert(!ExtraSymbolsToClaim.count(InternedName) &&
248 "Duplicate symbol to claim?");
249 ExtraSymbolsToClaim[InternedName] = Flags;
250 }
251 }
252
253 for (auto *Sym : G.absolute_symbols())
254 if (Sym->hasName() && Sym->getScope() != Scope::Local) {
255 auto InternedName = ES.intern(Sym->getName());
256 auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple());
257 auto Flags = getJITSymbolFlagsForSymbol(*Sym);
258 InternedResult[InternedName] = {Ptr, Flags};
259 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
260 assert(!ExtraSymbolsToClaim.count(InternedName) &&
261 "Duplicate symbol to claim?");
262 ExtraSymbolsToClaim[InternedName] = Flags;
263 }
264 }
265
266 if (!ExtraSymbolsToClaim.empty())
267 if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim))
268 return Err;
269
270 {
271
272 // Check that InternedResult matches up with MR->getSymbols(), overriding
273 // flags if requested.
274 // This guards against faulty transformations / compilers / object caches.
275
276 // First check that there aren't any missing symbols.
277 size_t NumMaterializationSideEffectsOnlySymbols = 0;
278 SymbolNameVector ExtraSymbols;
279 SymbolNameVector MissingSymbols;
280 for (auto &KV : MR->getSymbols()) {
281
282 auto I = InternedResult.find(KV.first);
283
284 // If this is a materialization-side-effects only symbol then bump
285 // the counter and make sure it's *not* defined, otherwise make
286 // sure that it is defined.
287 if (KV.second.hasMaterializationSideEffectsOnly()) {
288 ++NumMaterializationSideEffectsOnlySymbols;
289 if (I != InternedResult.end())
290 ExtraSymbols.push_back(KV.first);
291 continue;
292 } else if (I == InternedResult.end())
293 MissingSymbols.push_back(KV.first);
294 else if (Layer.OverrideObjectFlags)
295 I->second.setFlags(KV.second);
296 }
297
298 // If there were missing symbols then report the error.
299 if (!MissingSymbols.empty())
300 return make_error<MissingSymbolDefinitions>(
301 Layer.getExecutionSession().getSymbolStringPool(), G.getName(),
302 std::move(MissingSymbols));
303
304 // If there are more definitions than expected, add them to the
305 // ExtraSymbols vector.
306 if (InternedResult.size() >
307 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
308 for (auto &KV : InternedResult)
309 if (!MR->getSymbols().count(KV.first))
310 ExtraSymbols.push_back(KV.first);
311 }
312
313 // If there were extra definitions then report the error.
314 if (!ExtraSymbols.empty())
315 return make_error<UnexpectedSymbolDefinitions>(
316 Layer.getExecutionSession().getSymbolStringPool(), G.getName(),
317 std::move(ExtraSymbols));
318 }
319
320 if (auto Err = MR->notifyResolved(InternedResult))
321 return Err;
322
323 notifyLoaded();
324 return Error::success();
325 }
326
notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A)327 void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override {
328 if (auto Err = notifyEmitted(std::move(A))) {
329 Layer.getExecutionSession().reportError(std::move(Err));
330 MR->failMaterialization();
331 return;
332 }
333 if (auto Err = MR->notifyEmitted(SymbolDepGroups)) {
334 Layer.getExecutionSession().reportError(std::move(Err));
335 MR->failMaterialization();
336 }
337 }
338
getMarkLivePass(const Triple & TT) const339 LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override {
340 return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); };
341 }
342
modifyPassConfig(LinkGraph & LG,PassConfiguration & Config)343 Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override {
344 // Add passes to mark duplicate defs as should-discard, and to walk the
345 // link graph to build the symbol dependence graph.
346 Config.PrePrunePasses.push_back([this](LinkGraph &G) {
347 return claimOrExternalizeWeakAndCommonSymbols(G);
348 });
349
350 for (auto &P : Plugins)
351 P->modifyPassConfig(*MR, LG, Config);
352
353 Config.PreFixupPasses.push_back(
354 [this](LinkGraph &G) { return registerDependencies(G); });
355
356 return Error::success();
357 }
358
notifyLoaded()359 void notifyLoaded() {
360 for (auto &P : Plugins)
361 P->notifyLoaded(*MR);
362 }
363
notifyEmitted(jitlink::JITLinkMemoryManager::FinalizedAlloc FA)364 Error notifyEmitted(jitlink::JITLinkMemoryManager::FinalizedAlloc FA) {
365 Error Err = Error::success();
366 for (auto &P : Plugins)
367 Err = joinErrors(std::move(Err), P->notifyEmitted(*MR));
368
369 if (Err) {
370 if (FA)
371 Err =
372 joinErrors(std::move(Err), Layer.MemMgr.deallocate(std::move(FA)));
373 return Err;
374 }
375
376 if (FA)
377 return Layer.recordFinalizedAlloc(*MR, std::move(FA));
378
379 return Error::success();
380 }
381
382 private:
383 // Symbol name dependencies:
384 // Internal: Defined in this graph.
385 // External: Defined externally.
386 struct BlockSymbolDependencies {
387 SymbolNameSet Internal, External;
388 };
389
390 // Lazily populated map of blocks to BlockSymbolDependencies values.
391 class BlockDependenciesMap {
392 public:
BlockDependenciesMap(ExecutionSession & ES,DenseMap<const Block *,DenseSet<Block * >> BlockDeps)393 BlockDependenciesMap(ExecutionSession &ES,
394 DenseMap<const Block *, DenseSet<Block *>> BlockDeps)
395 : ES(ES), BlockDeps(std::move(BlockDeps)) {}
396
operator [](const Block & B)397 const BlockSymbolDependencies &operator[](const Block &B) {
398 // Check the cache first.
399 auto I = BlockTransitiveDepsCache.find(&B);
400 if (I != BlockTransitiveDepsCache.end())
401 return I->second;
402
403 // No value. Populate the cache.
404 BlockSymbolDependencies BTDCacheVal;
405 auto BDI = BlockDeps.find(&B);
406 assert(BDI != BlockDeps.end() && "No block dependencies");
407
408 for (auto *BDep : BDI->second) {
409 auto &BID = getBlockImmediateDeps(*BDep);
410 for (auto &ExternalDep : BID.External)
411 BTDCacheVal.External.insert(ExternalDep);
412 for (auto &InternalDep : BID.Internal)
413 BTDCacheVal.Internal.insert(InternalDep);
414 }
415
416 return BlockTransitiveDepsCache
417 .insert(std::make_pair(&B, std::move(BTDCacheVal)))
418 .first->second;
419 }
420
getInternedName(Symbol & Sym)421 SymbolStringPtr &getInternedName(Symbol &Sym) {
422 auto I = NameCache.find(&Sym);
423 if (I != NameCache.end())
424 return I->second;
425
426 return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.getName())))
427 .first->second;
428 }
429
430 private:
getBlockImmediateDeps(Block & B)431 BlockSymbolDependencies &getBlockImmediateDeps(Block &B) {
432 // Check the cache first.
433 auto I = BlockImmediateDepsCache.find(&B);
434 if (I != BlockImmediateDepsCache.end())
435 return I->second;
436
437 BlockSymbolDependencies BIDCacheVal;
438 for (auto &E : B.edges()) {
439 auto &Tgt = E.getTarget();
440 if (Tgt.getScope() != Scope::Local) {
441 if (Tgt.isExternal()) {
442 if (Tgt.getAddress() || !Tgt.isWeaklyReferenced())
443 BIDCacheVal.External.insert(getInternedName(Tgt));
444 } else
445 BIDCacheVal.Internal.insert(getInternedName(Tgt));
446 }
447 }
448
449 return BlockImmediateDepsCache
450 .insert(std::make_pair(&B, std::move(BIDCacheVal)))
451 .first->second;
452 }
453
454 ExecutionSession &ES;
455 DenseMap<const Block *, DenseSet<Block *>> BlockDeps;
456 DenseMap<const Symbol *, SymbolStringPtr> NameCache;
457 DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache;
458 DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache;
459 };
460
claimOrExternalizeWeakAndCommonSymbols(LinkGraph & G)461 Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) {
462 auto &ES = Layer.getExecutionSession();
463
464 SymbolFlagsMap NewSymbolsToClaim;
465 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
466
467 auto ProcessSymbol = [&](Symbol *Sym) {
468 if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak &&
469 Sym->getScope() != Scope::Local) {
470 auto Name = ES.intern(Sym->getName());
471 if (!MR->getSymbols().count(ES.intern(Sym->getName()))) {
472 NewSymbolsToClaim[Name] =
473 getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak;
474 NameToSym.push_back(std::make_pair(std::move(Name), Sym));
475 }
476 }
477 };
478
479 for (auto *Sym : G.defined_symbols())
480 ProcessSymbol(Sym);
481 for (auto *Sym : G.absolute_symbols())
482 ProcessSymbol(Sym);
483
484 // Attempt to claim all weak defs that we're not already responsible for.
485 // This may fail if the resource tracker has become defunct, but should
486 // always succeed otherwise.
487 if (auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim)))
488 return Err;
489
490 // Walk the list of symbols that we just tried to claim. Symbols that we're
491 // responsible for are marked live. Symbols that we're not responsible for
492 // are turned into external references.
493 for (auto &KV : NameToSym) {
494 if (MR->getSymbols().count(KV.first))
495 KV.second->setLive(true);
496 else
497 G.makeExternal(*KV.second);
498 }
499
500 return Error::success();
501 }
502
markResponsibilitySymbolsLive(LinkGraph & G) const503 Error markResponsibilitySymbolsLive(LinkGraph &G) const {
504 auto &ES = Layer.getExecutionSession();
505 for (auto *Sym : G.defined_symbols())
506 if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName())))
507 Sym->setLive(true);
508 return Error::success();
509 }
510
registerDependencies(LinkGraph & G)511 Error registerDependencies(LinkGraph &G) {
512 auto &TargetJD = MR->getTargetJITDylib();
513 auto &ES = TargetJD.getExecutionSession();
514 auto BlockDeps = computeBlockNonLocalDeps(G);
515
516 DenseSet<Block *> BlockDepsProcessed;
517 DenseMap<Block *, SymbolDependenceGroup> DepGroupForBlock;
518
519 // Compute dependencies for symbols defined in the JITLink graph.
520 for (auto *Sym : G.defined_symbols()) {
521
522 // Skip local symbols.
523 if (Sym->getScope() == Scope::Local)
524 continue;
525 assert(Sym->hasName() &&
526 "Defined non-local jitlink::Symbol should have a name");
527
528 auto &BDeps = BlockDeps[Sym->getBlock()];
529
530 // Skip symbols in blocks that don't depend on anything.
531 if (BDeps.Internal.empty() && BDeps.External.empty())
532 continue;
533
534 SymbolDependenceGroup &SDG = DepGroupForBlock[&Sym->getBlock()];
535 SDG.Symbols.insert(ES.intern(Sym->getName()));
536
537 if (!BlockDepsProcessed.count(&Sym->getBlock())) {
538 BlockDepsProcessed.insert(&Sym->getBlock());
539
540 if (!BDeps.Internal.empty())
541 SDG.Dependencies[&TargetJD] = BDeps.Internal;
542 for (auto &Dep : BDeps.External) {
543 auto DepSrcItr = SymbolSourceJDs.find(NonOwningSymbolStringPtr(Dep));
544 if (DepSrcItr != SymbolSourceJDs.end())
545 SDG.Dependencies[DepSrcItr->second].insert(Dep);
546 }
547 }
548 }
549
550 SymbolDependenceGroup SynthSDG;
551
552 for (auto &P : Plugins) {
553 auto SynthDeps = P->getSyntheticSymbolDependencies(*MR);
554 if (SynthDeps.empty())
555 continue;
556
557 DenseSet<Block *> BlockVisited;
558 for (auto &[Name, DepSyms] : SynthDeps) {
559 SynthSDG.Symbols.insert(Name);
560 for (auto *Sym : DepSyms) {
561 if (Sym->getScope() == Scope::Local) {
562 auto &BDeps = BlockDeps[Sym->getBlock()];
563 for (auto &S : BDeps.Internal)
564 SynthSDG.Dependencies[&TargetJD].insert(S);
565 for (auto &S : BDeps.External) {
566 auto DepSrcItr =
567 SymbolSourceJDs.find(NonOwningSymbolStringPtr(S));
568 if (DepSrcItr != SymbolSourceJDs.end())
569 SynthSDG.Dependencies[DepSrcItr->second].insert(S);
570 }
571 } else {
572 auto SymName = ES.intern(Sym->getName());
573 if (Sym->isExternal()) {
574 assert(SymbolSourceJDs.count(NonOwningSymbolStringPtr(SymName)) &&
575 "External symbol source entry missing");
576 SynthSDG
577 .Dependencies[SymbolSourceJDs[NonOwningSymbolStringPtr(
578 SymName)]]
579 .insert(SymName);
580 } else
581 SynthSDG.Dependencies[&TargetJD].insert(SymName);
582 }
583 }
584 }
585 }
586
587 // Transfer SDGs to SymbolDepGroups.
588 DepGroupForBlock.reserve(DepGroupForBlock.size() + 1);
589 for (auto &[B, SDG] : DepGroupForBlock) {
590 assert(!SDG.Symbols.empty() && "SymbolDependenceGroup covers no symbols");
591 if (!SDG.Dependencies.empty())
592 SymbolDepGroups.push_back(std::move(SDG));
593 }
594 if (!SynthSDG.Symbols.empty() && !SynthSDG.Dependencies.empty())
595 SymbolDepGroups.push_back(std::move(SynthSDG));
596
597 return Error::success();
598 }
599
computeBlockNonLocalDeps(LinkGraph & G)600 BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) {
601 // First calculate the reachable-via-non-local-symbol blocks for each block.
602 struct BlockInfo {
603 DenseSet<Block *> Dependencies;
604 DenseSet<Block *> Dependants;
605 bool DependenciesChanged = true;
606 };
607 DenseMap<Block *, BlockInfo> BlockInfos;
608 SmallVector<Block *> WorkList;
609
610 // Pre-allocate map entries. This prevents any iterator/reference
611 // invalidation in the next loop.
612 for (auto *B : G.blocks())
613 (void)BlockInfos[B];
614
615 // Build initial worklist, record block dependencies/dependants and
616 // non-local symbol dependencies.
617 for (auto *B : G.blocks()) {
618 auto &BI = BlockInfos[B];
619 for (auto &E : B->edges()) {
620 if (E.getTarget().getScope() == Scope::Local &&
621 !E.getTarget().isAbsolute()) {
622 auto &TgtB = E.getTarget().getBlock();
623 if (&TgtB != B) {
624 BI.Dependencies.insert(&TgtB);
625 BlockInfos[&TgtB].Dependants.insert(B);
626 }
627 }
628 }
629 }
630
631 // Add blocks with both dependants and dependencies to the worklist to
632 // propagate dependencies to dependants.
633 for (auto &[B, BI] : BlockInfos) {
634 if (!BI.Dependants.empty() && !BI.Dependencies.empty())
635 WorkList.push_back(B);
636 }
637
638 // Propagate block-level dependencies through the block-dependence graph.
639 while (!WorkList.empty()) {
640 auto *B = WorkList.pop_back_val();
641
642 auto &BI = BlockInfos[B];
643 assert(BI.DependenciesChanged &&
644 "Block in worklist has unchanged dependencies");
645 BI.DependenciesChanged = false;
646 for (auto *Dependant : BI.Dependants) {
647 auto &DependantBI = BlockInfos[Dependant];
648 for (auto *Dependency : BI.Dependencies) {
649 if (Dependant != Dependency &&
650 DependantBI.Dependencies.insert(Dependency).second)
651 if (!DependantBI.DependenciesChanged) {
652 DependantBI.DependenciesChanged = true;
653 WorkList.push_back(Dependant);
654 }
655 }
656 }
657 }
658
659 DenseMap<const Block *, DenseSet<Block *>> BlockDeps;
660 for (auto &KV : BlockInfos)
661 BlockDeps[KV.first] = std::move(KV.second.Dependencies);
662
663 return BlockDependenciesMap(Layer.getExecutionSession(),
664 std::move(BlockDeps));
665 }
666
667 ObjectLinkingLayer &Layer;
668 std::vector<std::shared_ptr<ObjectLinkingLayer::Plugin>> Plugins;
669 std::unique_ptr<MaterializationResponsibility> MR;
670 std::unique_ptr<MemoryBuffer> ObjBuffer;
671 DenseMap<Block *, SymbolNameSet> ExternalBlockDeps;
672 DenseMap<Block *, SymbolNameSet> InternalBlockDeps;
673 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs;
674 std::vector<SymbolDependenceGroup> SymbolDepGroups;
675 };
676
677 ObjectLinkingLayer::Plugin::~Plugin() = default;
678
679 char ObjectLinkingLayer::ID;
680
681 using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>;
682
ObjectLinkingLayer(ExecutionSession & ES)683 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES)
684 : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) {
685 ES.registerResourceManager(*this);
686 }
687
ObjectLinkingLayer(ExecutionSession & ES,JITLinkMemoryManager & MemMgr)688 ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES,
689 JITLinkMemoryManager &MemMgr)
690 : BaseT(ES), MemMgr(MemMgr) {
691 ES.registerResourceManager(*this);
692 }
693
ObjectLinkingLayer(ExecutionSession & ES,std::unique_ptr<JITLinkMemoryManager> MemMgr)694 ObjectLinkingLayer::ObjectLinkingLayer(
695 ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr)
696 : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) {
697 ES.registerResourceManager(*this);
698 }
699
~ObjectLinkingLayer()700 ObjectLinkingLayer::~ObjectLinkingLayer() {
701 assert(Allocs.empty() && "Layer destroyed with resources still attached");
702 getExecutionSession().deregisterResourceManager(*this);
703 }
704
add(ResourceTrackerSP RT,std::unique_ptr<LinkGraph> G)705 Error ObjectLinkingLayer::add(ResourceTrackerSP RT,
706 std::unique_ptr<LinkGraph> G) {
707 auto &JD = RT->getJITDylib();
708 return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)),
709 std::move(RT));
710 }
711
emit(std::unique_ptr<MaterializationResponsibility> R,std::unique_ptr<MemoryBuffer> O)712 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
713 std::unique_ptr<MemoryBuffer> O) {
714 assert(O && "Object must not be null");
715 MemoryBufferRef ObjBuffer = O->getMemBufferRef();
716
717 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
718 *this, std::move(R), std::move(O));
719 if (auto G = createLinkGraphFromObject(ObjBuffer)) {
720 Ctx->notifyMaterializing(**G);
721 link(std::move(*G), std::move(Ctx));
722 } else {
723 Ctx->notifyFailed(G.takeError());
724 }
725 }
726
emit(std::unique_ptr<MaterializationResponsibility> R,std::unique_ptr<LinkGraph> G)727 void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
728 std::unique_ptr<LinkGraph> G) {
729 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
730 *this, std::move(R), nullptr);
731 Ctx->notifyMaterializing(*G);
732 link(std::move(G), std::move(Ctx));
733 }
734
recordFinalizedAlloc(MaterializationResponsibility & MR,FinalizedAlloc FA)735 Error ObjectLinkingLayer::recordFinalizedAlloc(
736 MaterializationResponsibility &MR, FinalizedAlloc FA) {
737 auto Err = MR.withResourceKeyDo(
738 [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); });
739
740 if (Err)
741 Err = joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA)));
742
743 return Err;
744 }
745
handleRemoveResources(JITDylib & JD,ResourceKey K)746 Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) {
747
748 {
749 Error Err = Error::success();
750 for (auto &P : Plugins)
751 Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K));
752 if (Err)
753 return Err;
754 }
755
756 std::vector<FinalizedAlloc> AllocsToRemove;
757 getExecutionSession().runSessionLocked([&] {
758 auto I = Allocs.find(K);
759 if (I != Allocs.end()) {
760 std::swap(AllocsToRemove, I->second);
761 Allocs.erase(I);
762 }
763 });
764
765 if (AllocsToRemove.empty())
766 return Error::success();
767
768 return MemMgr.deallocate(std::move(AllocsToRemove));
769 }
770
handleTransferResources(JITDylib & JD,ResourceKey DstKey,ResourceKey SrcKey)771 void ObjectLinkingLayer::handleTransferResources(JITDylib &JD,
772 ResourceKey DstKey,
773 ResourceKey SrcKey) {
774 auto I = Allocs.find(SrcKey);
775 if (I != Allocs.end()) {
776 auto &SrcAllocs = I->second;
777 auto &DstAllocs = Allocs[DstKey];
778 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
779 for (auto &Alloc : SrcAllocs)
780 DstAllocs.push_back(std::move(Alloc));
781
782 // Erase SrcKey entry using value rather than iterator I: I may have been
783 // invalidated when we looked up DstKey.
784 Allocs.erase(SrcKey);
785 }
786
787 for (auto &P : Plugins)
788 P->notifyTransferringResources(JD, DstKey, SrcKey);
789 }
790
EHFrameRegistrationPlugin(ExecutionSession & ES,std::unique_ptr<EHFrameRegistrar> Registrar)791 EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
792 ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar)
793 : ES(ES), Registrar(std::move(Registrar)) {}
794
modifyPassConfig(MaterializationResponsibility & MR,LinkGraph & G,PassConfiguration & PassConfig)795 void EHFrameRegistrationPlugin::modifyPassConfig(
796 MaterializationResponsibility &MR, LinkGraph &G,
797 PassConfiguration &PassConfig) {
798
799 PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass(
800 G.getTargetTriple(), [this, &MR](ExecutorAddr Addr, size_t Size) {
801 if (Addr) {
802 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
803 assert(!InProcessLinks.count(&MR) &&
804 "Link for MR already being tracked?");
805 InProcessLinks[&MR] = {Addr, Size};
806 }
807 }));
808 }
809
notifyEmitted(MaterializationResponsibility & MR)810 Error EHFrameRegistrationPlugin::notifyEmitted(
811 MaterializationResponsibility &MR) {
812
813 ExecutorAddrRange EmittedRange;
814 {
815 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
816
817 auto EHFrameRangeItr = InProcessLinks.find(&MR);
818 if (EHFrameRangeItr == InProcessLinks.end())
819 return Error::success();
820
821 EmittedRange = EHFrameRangeItr->second;
822 assert(EmittedRange.Start && "eh-frame addr to register can not be null");
823 InProcessLinks.erase(EHFrameRangeItr);
824 }
825
826 if (auto Err = MR.withResourceKeyDo(
827 [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); }))
828 return Err;
829
830 return Registrar->registerEHFrames(EmittedRange);
831 }
832
notifyFailed(MaterializationResponsibility & MR)833 Error EHFrameRegistrationPlugin::notifyFailed(
834 MaterializationResponsibility &MR) {
835 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
836 InProcessLinks.erase(&MR);
837 return Error::success();
838 }
839
notifyRemovingResources(JITDylib & JD,ResourceKey K)840 Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD,
841 ResourceKey K) {
842 std::vector<ExecutorAddrRange> RangesToRemove;
843
844 ES.runSessionLocked([&] {
845 auto I = EHFrameRanges.find(K);
846 if (I != EHFrameRanges.end()) {
847 RangesToRemove = std::move(I->second);
848 EHFrameRanges.erase(I);
849 }
850 });
851
852 Error Err = Error::success();
853 while (!RangesToRemove.empty()) {
854 auto RangeToRemove = RangesToRemove.back();
855 RangesToRemove.pop_back();
856 assert(RangeToRemove.Start && "Untracked eh-frame range must not be null");
857 Err = joinErrors(std::move(Err),
858 Registrar->deregisterEHFrames(RangeToRemove));
859 }
860
861 return Err;
862 }
863
notifyTransferringResources(JITDylib & JD,ResourceKey DstKey,ResourceKey SrcKey)864 void EHFrameRegistrationPlugin::notifyTransferringResources(
865 JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {
866 auto SI = EHFrameRanges.find(SrcKey);
867 if (SI == EHFrameRanges.end())
868 return;
869
870 auto DI = EHFrameRanges.find(DstKey);
871 if (DI != EHFrameRanges.end()) {
872 auto &SrcRanges = SI->second;
873 auto &DstRanges = DI->second;
874 DstRanges.reserve(DstRanges.size() + SrcRanges.size());
875 for (auto &SrcRange : SrcRanges)
876 DstRanges.push_back(std::move(SrcRange));
877 EHFrameRanges.erase(SI);
878 } else {
879 // We need to move SrcKey's ranges over without invalidating the SI
880 // iterator.
881 auto Tmp = std::move(SI->second);
882 EHFrameRanges.erase(SI);
883 EHFrameRanges[DstKey] = std::move(Tmp);
884 }
885 }
886
887 } // End namespace orc.
888 } // End namespace llvm.
889