xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp (revision e6bfd18d21b225af6a0ed67ceeaf1293b7b9eba5) 
1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10 
11 #include "llvm/BinaryFormat/ELF.h"
12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
14 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
15 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
16 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
17 #include "llvm/Support/BinaryByteStream.h"
18 #include "llvm/Support/Debug.h"
19 
20 #define DEBUG_TYPE "orc"
21 
22 using namespace llvm;
23 using namespace llvm::orc;
24 using namespace llvm::orc::shared;
25 
26 namespace {
27 
28 class DSOHandleMaterializationUnit : public MaterializationUnit {
29 public:
30   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
31                                const SymbolStringPtr &DSOHandleSymbol)
32       : MaterializationUnit(
33             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
34         ENP(ENP) {}
35 
36   StringRef getName() const override { return "DSOHandleMU"; }
37 
38   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
39     unsigned PointerSize;
40     support::endianness Endianness;
41     jitlink::Edge::Kind EdgeKind;
42     const auto &TT =
43         ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
44 
45     switch (TT.getArch()) {
46     case Triple::x86_64:
47       PointerSize = 8;
48       Endianness = support::endianness::little;
49       EdgeKind = jitlink::x86_64::Pointer64;
50       break;
51     case Triple::aarch64:
52       PointerSize = 8;
53       Endianness = support::endianness::little;
54       EdgeKind = jitlink::aarch64::Pointer64;
55       break;
56     default:
57       llvm_unreachable("Unrecognized architecture");
58     }
59 
60     // void *__dso_handle = &__dso_handle;
61     auto G = std::make_unique<jitlink::LinkGraph>(
62         "<DSOHandleMU>", TT, PointerSize, Endianness,
63         jitlink::getGenericEdgeKindName);
64     auto &DSOHandleSection =
65         G->createSection(".data.__dso_handle", jitlink::MemProt::Read);
66     auto &DSOHandleBlock = G->createContentBlock(
67         DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(),
68         8, 0);
69     auto &DSOHandleSymbol = G->addDefinedSymbol(
70         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
71         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
72     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
73 
74     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
75   }
76 
77   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
78 
79 private:
80   static MaterializationUnit::Interface
81   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
82                                   const SymbolStringPtr &DSOHandleSymbol) {
83     SymbolFlagsMap SymbolFlags;
84     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
85     return MaterializationUnit::Interface(std::move(SymbolFlags),
86                                           DSOHandleSymbol);
87   }
88 
89   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
90     static const char Content[8] = {0};
91     assert(PointerSize <= sizeof Content);
92     return {Content, PointerSize};
93   }
94 
95   ELFNixPlatform &ENP;
96 };
97 
98 StringRef EHFrameSectionName = ".eh_frame";
99 StringRef InitArrayFuncSectionName = ".init_array";
100 
101 StringRef ThreadBSSSectionName = ".tbss";
102 StringRef ThreadDataSectionName = ".tdata";
103 
104 } // end anonymous namespace
105 
106 namespace llvm {
107 namespace orc {
108 
109 Expected<std::unique_ptr<ELFNixPlatform>>
110 ELFNixPlatform::Create(ExecutionSession &ES,
111                        ObjectLinkingLayer &ObjLinkingLayer,
112                        JITDylib &PlatformJD, const char *OrcRuntimePath,
113                        Optional<SymbolAliasMap> RuntimeAliases) {
114 
115   auto &EPC = ES.getExecutorProcessControl();
116 
117   // If the target is not supported then bail out immediately.
118   if (!supportedTarget(EPC.getTargetTriple()))
119     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
120                                        EPC.getTargetTriple().str(),
121                                    inconvertibleErrorCode());
122 
123   // Create default aliases if the caller didn't supply any.
124   if (!RuntimeAliases) {
125     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
126     if (!StandardRuntimeAliases)
127       return StandardRuntimeAliases.takeError();
128     RuntimeAliases = std::move(*StandardRuntimeAliases);
129   }
130 
131   // Define the aliases.
132   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
133     return std::move(Err);
134 
135   // Add JIT-dispatch function support symbols.
136   if (auto Err = PlatformJD.define(absoluteSymbols(
137           {{ES.intern("__orc_rt_jit_dispatch"),
138             {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
139              JITSymbolFlags::Exported}},
140            {ES.intern("__orc_rt_jit_dispatch_ctx"),
141             {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
142              JITSymbolFlags::Exported}}})))
143     return std::move(Err);
144 
145   // Create a generator for the ORC runtime archive.
146   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
147       ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
148   if (!OrcRuntimeArchiveGenerator)
149     return OrcRuntimeArchiveGenerator.takeError();
150 
151   // Create the instance.
152   Error Err = Error::success();
153   auto P = std::unique_ptr<ELFNixPlatform>(
154       new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD,
155                          std::move(*OrcRuntimeArchiveGenerator), Err));
156   if (Err)
157     return std::move(Err);
158   return std::move(P);
159 }
160 
161 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
162   return JD.define(
163       std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
164 }
165 
166 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
167   return Error::success();
168 }
169 
170 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
171                                    const MaterializationUnit &MU) {
172   auto &JD = RT.getJITDylib();
173   const auto &InitSym = MU.getInitializerSymbol();
174   if (!InitSym)
175     return Error::success();
176 
177   RegisteredInitSymbols[&JD].add(InitSym,
178                                  SymbolLookupFlags::WeaklyReferencedSymbol);
179   LLVM_DEBUG({
180     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
181            << " for MU " << MU.getName() << "\n";
182   });
183   return Error::success();
184 }
185 
186 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
187   llvm_unreachable("Not supported yet");
188 }
189 
190 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
191                        ArrayRef<std::pair<const char *, const char *>> AL) {
192   for (auto &KV : AL) {
193     auto AliasName = ES.intern(KV.first);
194     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
195     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
196                                      JITSymbolFlags::Exported};
197   }
198 }
199 
200 Expected<SymbolAliasMap>
201 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
202                                         JITDylib &PlatformJD) {
203   SymbolAliasMap Aliases;
204   addAliases(ES, Aliases, requiredCXXAliases());
205   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
206 
207   // Determine whether or not the libunwind extended-API function for
208   // dynamically registering an entire .eh_frame section is available.
209   // If it is not, we assume that libgcc_s is being used, and alias to
210   // its __register_frame with the same functionality.
211   auto RTRegisterFrame = ES.intern("__orc_rt_register_eh_frame_section");
212   auto LibUnwindRegisterFrame = ES.intern("__unw_add_dynamic_eh_frame_section");
213   auto RTDeregisterFrame = ES.intern("__orc_rt_deregister_eh_frame_section");
214   auto LibUnwindDeregisterFrame =
215       ES.intern("__unw_remove_dynamic_eh_frame_section");
216   auto SM = ES.lookup(makeJITDylibSearchOrder(&PlatformJD),
217                       SymbolLookupSet()
218                           .add(LibUnwindRegisterFrame,
219                                SymbolLookupFlags::WeaklyReferencedSymbol)
220                           .add(LibUnwindDeregisterFrame,
221                                SymbolLookupFlags::WeaklyReferencedSymbol));
222   if (!SM) { // Weak-ref means no "missing symbol" errors, so this must be
223              // something more serious that we should report.
224     return SM.takeError();
225   } else if (SM->size() == 2) {
226     LLVM_DEBUG({
227       dbgs() << "Using libunwind " << LibUnwindRegisterFrame
228              << " for unwind info registration\n";
229     });
230     Aliases[std::move(RTRegisterFrame)] = {LibUnwindRegisterFrame,
231                                            JITSymbolFlags::Exported};
232     Aliases[std::move(RTDeregisterFrame)] = {LibUnwindDeregisterFrame,
233                                              JITSymbolFlags::Exported};
234   } else {
235     // Since LLVM libunwind is not present, we assume that unwinding
236     // is provided by libgcc
237     LLVM_DEBUG({
238       dbgs() << "Using libgcc __register_frame"
239              << " for unwind info registration\n";
240     });
241     Aliases[std::move(RTRegisterFrame)] = {ES.intern("__register_frame"),
242                                            JITSymbolFlags::Exported};
243     Aliases[std::move(RTDeregisterFrame)] = {ES.intern("__deregister_frame"),
244                                              JITSymbolFlags::Exported};
245   }
246 
247   return Aliases;
248 }
249 
250 ArrayRef<std::pair<const char *, const char *>>
251 ELFNixPlatform::requiredCXXAliases() {
252   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
253       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
254       {"atexit", "__orc_rt_elfnix_atexit"}};
255 
256   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
257 }
258 
259 ArrayRef<std::pair<const char *, const char *>>
260 ELFNixPlatform::standardRuntimeUtilityAliases() {
261   static const std::pair<const char *, const char *>
262       StandardRuntimeUtilityAliases[] = {
263           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
264           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
265           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
266           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
267           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
268           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
269 
270   return ArrayRef<std::pair<const char *, const char *>>(
271       StandardRuntimeUtilityAliases);
272 }
273 
274 bool ELFNixPlatform::isInitializerSection(StringRef SecName) {
275   if (SecName.consume_front(InitArrayFuncSectionName) &&
276       (SecName.empty() || SecName[0] == '.'))
277     return true;
278   return false;
279 }
280 
281 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
282   switch (TT.getArch()) {
283   case Triple::x86_64:
284   case Triple::aarch64:
285     return true;
286   default:
287     return false;
288   }
289 }
290 
291 ELFNixPlatform::ELFNixPlatform(
292     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
293     JITDylib &PlatformJD,
294     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
295     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
296       DSOHandleSymbol(ES.intern("__dso_handle")) {
297   ErrorAsOutParameter _(&Err);
298 
299   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
300 
301   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
302 
303   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
304   // the platform now), so set it up.
305   if (auto E2 = setupJITDylib(PlatformJD)) {
306     Err = std::move(E2);
307     return;
308   }
309 
310   RegisteredInitSymbols[&PlatformJD].add(
311       DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
312 
313   // Associate wrapper function tags with JIT-side function implementations.
314   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
315     Err = std::move(E2);
316     return;
317   }
318 
319   // Lookup addresses of runtime functions callable by the platform,
320   // call the platform bootstrap function to initialize the platform-state
321   // object in the executor.
322   if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
323     Err = std::move(E2);
324     return;
325   }
326 }
327 
328 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
329   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
330 
331   using GetInitializersSPSSig =
332       SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
333   WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
334       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
335           this, &ELFNixPlatform::rt_getInitializers);
336 
337   using GetDeinitializersSPSSig =
338       SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
339   WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
340       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
341           this, &ELFNixPlatform::rt_getDeinitializers);
342 
343   using LookupSymbolSPSSig =
344       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
345   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
346       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
347                                               &ELFNixPlatform::rt_lookupSymbol);
348 
349   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
350 }
351 
352 void ELFNixPlatform::getInitializersBuildSequencePhase(
353     SendInitializerSequenceFn SendResult, JITDylib &JD,
354     std::vector<JITDylibSP> DFSLinkOrder) {
355   ELFNixJITDylibInitializerSequence FullInitSeq;
356   {
357     std::lock_guard<std::mutex> Lock(PlatformMutex);
358     for (auto &InitJD : reverse(DFSLinkOrder)) {
359       LLVM_DEBUG({
360         dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
361                << "\" to sequence\n";
362       });
363       auto ISItr = InitSeqs.find(InitJD.get());
364       if (ISItr != InitSeqs.end()) {
365         FullInitSeq.emplace_back(std::move(ISItr->second));
366         InitSeqs.erase(ISItr);
367       }
368     }
369   }
370 
371   SendResult(std::move(FullInitSeq));
372 }
373 
374 void ELFNixPlatform::getInitializersLookupPhase(
375     SendInitializerSequenceFn SendResult, JITDylib &JD) {
376 
377   auto DFSLinkOrder = JD.getDFSLinkOrder();
378   if (!DFSLinkOrder) {
379     SendResult(DFSLinkOrder.takeError());
380     return;
381   }
382 
383   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
384   ES.runSessionLocked([&]() {
385     for (auto &InitJD : *DFSLinkOrder) {
386       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
387       if (RISItr != RegisteredInitSymbols.end()) {
388         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
389         RegisteredInitSymbols.erase(RISItr);
390       }
391     }
392   });
393 
394   // If there are no further init symbols to look up then move on to the next
395   // phase.
396   if (NewInitSymbols.empty()) {
397     getInitializersBuildSequencePhase(std::move(SendResult), JD,
398                                       std::move(*DFSLinkOrder));
399     return;
400   }
401 
402   // Otherwise issue a lookup and re-run this phase when it completes.
403   lookupInitSymbolsAsync(
404       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
405         if (Err)
406           SendResult(std::move(Err));
407         else
408           getInitializersLookupPhase(std::move(SendResult), JD);
409       },
410       ES, std::move(NewInitSymbols));
411 }
412 
413 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
414                                         StringRef JDName) {
415   LLVM_DEBUG({
416     dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
417   });
418 
419   JITDylib *JD = ES.getJITDylibByName(JDName);
420   if (!JD) {
421     LLVM_DEBUG({
422       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
423     });
424     SendResult(make_error<StringError>("No JITDylib named " + JDName,
425                                        inconvertibleErrorCode()));
426     return;
427   }
428 
429   getInitializersLookupPhase(std::move(SendResult), *JD);
430 }
431 
432 void ELFNixPlatform::rt_getDeinitializers(
433     SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
434   LLVM_DEBUG({
435     dbgs() << "ELFNixPlatform::rt_getDeinitializers(\""
436            << formatv("{0:x}", Handle.getValue()) << "\")\n";
437   });
438 
439   JITDylib *JD = nullptr;
440 
441   {
442     std::lock_guard<std::mutex> Lock(PlatformMutex);
443     auto I = HandleAddrToJITDylib.find(Handle);
444     if (I != HandleAddrToJITDylib.end())
445       JD = I->second;
446   }
447 
448   if (!JD) {
449     LLVM_DEBUG({
450       dbgs() << "  No JITDylib for handle "
451              << formatv("{0:x}", Handle.getValue()) << "\n";
452     });
453     SendResult(make_error<StringError>("No JITDylib associated with handle " +
454                                            formatv("{0:x}", Handle.getValue()),
455                                        inconvertibleErrorCode()));
456     return;
457   }
458 
459   SendResult(ELFNixJITDylibDeinitializerSequence());
460 }
461 
462 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
463                                      ExecutorAddr Handle,
464                                      StringRef SymbolName) {
465   LLVM_DEBUG({
466     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\""
467            << formatv("{0:x}", Handle.getValue()) << "\")\n";
468   });
469 
470   JITDylib *JD = nullptr;
471 
472   {
473     std::lock_guard<std::mutex> Lock(PlatformMutex);
474     auto I = HandleAddrToJITDylib.find(Handle);
475     if (I != HandleAddrToJITDylib.end())
476       JD = I->second;
477   }
478 
479   if (!JD) {
480     LLVM_DEBUG({
481       dbgs() << "  No JITDylib for handle "
482              << formatv("{0:x}", Handle.getValue()) << "\n";
483     });
484     SendResult(make_error<StringError>("No JITDylib associated with handle " +
485                                            formatv("{0:x}", Handle.getValue()),
486                                        inconvertibleErrorCode()));
487     return;
488   }
489 
490   // Use functor class to work around XL build compiler issue on AIX.
491   class RtLookupNotifyComplete {
492   public:
493     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
494         : SendResult(std::move(SendResult)) {}
495     void operator()(Expected<SymbolMap> Result) {
496       if (Result) {
497         assert(Result->size() == 1 && "Unexpected result map count");
498         SendResult(ExecutorAddr(Result->begin()->second.getAddress()));
499       } else {
500         SendResult(Result.takeError());
501       }
502     }
503 
504   private:
505     SendSymbolAddressFn SendResult;
506   };
507 
508   ES.lookup(
509       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
510       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
511       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
512 }
513 
514 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
515 
516   std::pair<const char *, ExecutorAddr *> Symbols[] = {
517       {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
518       {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
519       {"__orc_rt_elfnix_register_object_sections",
520        &orc_rt_elfnix_register_object_sections},
521       {"__orc_rt_elfnix_create_pthread_key",
522        &orc_rt_elfnix_create_pthread_key}};
523 
524   SymbolLookupSet RuntimeSymbols;
525   std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
526   for (const auto &KV : Symbols) {
527     auto Name = ES.intern(KV.first);
528     RuntimeSymbols.add(Name);
529     AddrsToRecord.push_back({std::move(Name), KV.second});
530   }
531 
532   auto RuntimeSymbolAddrs = ES.lookup(
533       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
534   if (!RuntimeSymbolAddrs)
535     return RuntimeSymbolAddrs.takeError();
536 
537   for (const auto &KV : AddrsToRecord) {
538     auto &Name = KV.first;
539     assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
540     KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress());
541   }
542 
543   auto PJDDSOHandle = ES.lookup(
544       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
545   if (!PJDDSOHandle)
546     return PJDDSOHandle.takeError();
547 
548   if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
549           orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress()))
550     return Err;
551 
552   // FIXME: Ordering is fuzzy here. We're probably best off saying
553   // "behavior is undefined if code that uses the runtime is added before
554   // the platform constructor returns", then move all this to the constructor.
555   RuntimeBootstrapped = true;
556   std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
557   {
558     std::lock_guard<std::mutex> Lock(PlatformMutex);
559     DeferredPOSRs = std::move(BootstrapPOSRs);
560   }
561 
562   for (auto &D : DeferredPOSRs)
563     if (auto Err = registerPerObjectSections(D))
564       return Err;
565 
566   return Error::success();
567 }
568 
569 Error ELFNixPlatform::registerInitInfo(
570     JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
571 
572   std::unique_lock<std::mutex> Lock(PlatformMutex);
573 
574   ELFNixJITDylibInitializers *InitSeq = nullptr;
575   {
576     auto I = InitSeqs.find(&JD);
577     if (I == InitSeqs.end()) {
578       // If there's no init sequence entry yet then we need to look up the
579       // header symbol to force creation of one.
580       Lock.unlock();
581 
582       auto SearchOrder =
583           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
584       if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
585         return Err;
586 
587       Lock.lock();
588       I = InitSeqs.find(&JD);
589       assert(I != InitSeqs.end() &&
590              "Entry missing after header symbol lookup?");
591     }
592     InitSeq = &I->second;
593   }
594 
595   for (auto *Sec : InitSections) {
596     // FIXME: Avoid copy here.
597     jitlink::SectionRange R(*Sec);
598     InitSeq->InitSections[Sec->getName()].push_back(
599         {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())});
600   }
601 
602   return Error::success();
603 }
604 
605 Error ELFNixPlatform::registerPerObjectSections(
606     const ELFPerObjectSectionsToRegister &POSR) {
607 
608   if (!orc_rt_elfnix_register_object_sections)
609     return make_error<StringError>("Attempting to register per-object "
610                                    "sections, but runtime support has not "
611                                    "been loaded yet",
612                                    inconvertibleErrorCode());
613 
614   Error ErrResult = Error::success();
615   if (auto Err = ES.callSPSWrapper<shared::SPSError(
616                      SPSELFPerObjectSectionsToRegister)>(
617           orc_rt_elfnix_register_object_sections, ErrResult, POSR))
618     return Err;
619   return ErrResult;
620 }
621 
622 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
623   if (!orc_rt_elfnix_create_pthread_key)
624     return make_error<StringError>(
625         "Attempting to create pthread key in target, but runtime support has "
626         "not been loaded yet",
627         inconvertibleErrorCode());
628 
629   Expected<uint64_t> Result(0);
630   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
631           orc_rt_elfnix_create_pthread_key, Result))
632     return std::move(Err);
633   return Result;
634 }
635 
636 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
637     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
638     jitlink::PassConfiguration &Config) {
639 
640   // If the initializer symbol is the __dso_handle symbol then just add
641   // the DSO handle support passes.
642   if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
643     addDSOHandleSupportPasses(MR, Config);
644     // The DSOHandle materialization unit doesn't require any other
645     // support, so we can bail out early.
646     return;
647   }
648 
649   // If the object contains initializers then add passes to record them.
650   if (MR.getInitializerSymbol())
651     addInitializerSupportPasses(MR, Config);
652 
653   // Add passes for eh-frame and TLV support.
654   addEHAndTLVSupportPasses(MR, Config);
655 }
656 
657 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
658 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
659     MaterializationResponsibility &MR) {
660   std::lock_guard<std::mutex> Lock(PluginMutex);
661   auto I = InitSymbolDeps.find(&MR);
662   if (I != InitSymbolDeps.end()) {
663     SyntheticSymbolDependenciesMap Result;
664     Result[MR.getInitializerSymbol()] = std::move(I->second);
665     InitSymbolDeps.erase(&MR);
666     return Result;
667   }
668   return SyntheticSymbolDependenciesMap();
669 }
670 
671 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
672     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
673 
674   /// Preserve init sections.
675   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
676     if (auto Err = preserveInitSections(G, MR))
677       return Err;
678     return Error::success();
679   });
680 
681   Config.PostFixupPasses.push_back(
682       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
683         return registerInitSections(G, JD);
684       });
685 }
686 
687 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
688     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
689 
690   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
691                                             jitlink::LinkGraph &G) -> Error {
692     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
693       return Sym->getName() == *MP.DSOHandleSymbol;
694     });
695     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
696     {
697       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
698       auto HandleAddr = (*I)->getAddress();
699       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
700       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
701       MP.InitSeqs.insert(std::make_pair(
702           &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
703     }
704     return Error::success();
705   });
706 }
707 
708 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
709     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
710 
711   // Insert TLV lowering at the start of the PostPrunePasses, since we want
712   // it to run before GOT/PLT lowering.
713 
714   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
715   // pass has done. Because the TLS descriptor need to be allocate in GOT.
716   Config.PostPrunePasses.push_back(
717       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
718         return fixTLVSectionsAndEdges(G, JD);
719       });
720 
721   // Add a pass to register the final addresses of the eh-frame and TLV sections
722   // with the runtime.
723   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
724     ELFPerObjectSectionsToRegister POSR;
725 
726     if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
727       jitlink::SectionRange R(*EHFrameSection);
728       if (!R.empty())
729         POSR.EHFrameSection = {ExecutorAddr(R.getStart()),
730                                ExecutorAddr(R.getEnd())};
731     }
732 
733     // Get a pointer to the thread data section if there is one. It will be used
734     // below.
735     jitlink::Section *ThreadDataSection =
736         G.findSectionByName(ThreadDataSectionName);
737 
738     // Handle thread BSS section if there is one.
739     if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
740       // If there's already a thread data section in this graph then merge the
741       // thread BSS section content into it, otherwise just treat the thread
742       // BSS section as the thread data section.
743       if (ThreadDataSection)
744         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
745       else
746         ThreadDataSection = ThreadBSSSection;
747     }
748 
749     // Having merged thread BSS (if present) and thread data (if present),
750     // record the resulting section range.
751     if (ThreadDataSection) {
752       jitlink::SectionRange R(*ThreadDataSection);
753       if (!R.empty())
754         POSR.ThreadDataSection = {ExecutorAddr(R.getStart()),
755                                   ExecutorAddr(R.getEnd())};
756     }
757 
758     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
759 
760       // If we're still bootstrapping the runtime then just record this
761       // frame for now.
762       if (!MP.RuntimeBootstrapped) {
763         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
764         MP.BootstrapPOSRs.push_back(POSR);
765         return Error::success();
766       }
767 
768       // Otherwise register it immediately.
769       if (auto Err = MP.registerPerObjectSections(POSR))
770         return Err;
771     }
772 
773     return Error::success();
774   });
775 }
776 
777 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
778     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
779 
780   JITLinkSymbolSet InitSectionSymbols;
781   for (auto &InitSection : G.sections()) {
782     // Skip non-init sections.
783     if (!isInitializerSection(InitSection.getName()))
784       continue;
785 
786     // Make a pass over live symbols in the section: those blocks are already
787     // preserved.
788     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
789     for (auto &Sym : InitSection.symbols()) {
790       auto &B = Sym->getBlock();
791       if (Sym->isLive() && Sym->getOffset() == 0 &&
792           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
793         InitSectionSymbols.insert(Sym);
794         AlreadyLiveBlocks.insert(&B);
795       }
796     }
797 
798     // Add anonymous symbols to preserve any not-already-preserved blocks.
799     for (auto *B : InitSection.blocks())
800       if (!AlreadyLiveBlocks.count(B))
801         InitSectionSymbols.insert(
802             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
803   }
804 
805   if (!InitSectionSymbols.empty()) {
806     std::lock_guard<std::mutex> Lock(PluginMutex);
807     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
808   }
809 
810   return Error::success();
811 }
812 
813 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
814     jitlink::LinkGraph &G, JITDylib &JD) {
815 
816   SmallVector<jitlink::Section *> InitSections;
817 
818   LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; });
819 
820   for (auto &Sec : G.sections()) {
821     if (isInitializerSection(Sec.getName())) {
822       InitSections.push_back(&Sec);
823     }
824   }
825 
826   // Dump the scraped inits.
827   LLVM_DEBUG({
828     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
829     for (auto *Sec : InitSections) {
830       jitlink::SectionRange R(*Sec);
831       dbgs() << "  " << Sec->getName() << ": "
832              << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n";
833     }
834   });
835 
836   return MP.registerInitInfo(JD, InitSections);
837 }
838 
839 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
840     jitlink::LinkGraph &G, JITDylib &JD) {
841 
842   for (auto *Sym : G.external_symbols()) {
843     if (Sym->getName() == "__tls_get_addr") {
844       Sym->setName("___orc_rt_elfnix_tls_get_addr");
845     } else if (Sym->getName() == "__tlsdesc_resolver") {
846       Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
847     }
848   }
849 
850   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
851 
852   if (TLSInfoEntrySection) {
853     Optional<uint64_t> Key;
854     {
855       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
856       auto I = MP.JITDylibToPThreadKey.find(&JD);
857       if (I != MP.JITDylibToPThreadKey.end())
858         Key = I->second;
859     }
860     if (!Key) {
861       if (auto KeyOrErr = MP.createPThreadKey())
862         Key = *KeyOrErr;
863       else
864         return KeyOrErr.takeError();
865     }
866 
867     uint64_t PlatformKeyBits =
868         support::endian::byte_swap(*Key, G.getEndianness());
869 
870     for (auto *B : TLSInfoEntrySection->blocks()) {
871       // FIXME: The TLS descriptor byte length may different with different
872       // ISA
873       assert(B->getSize() == (G.getPointerSize() * 2) &&
874              "TLS descriptor must be 2 words length");
875       auto TLSInfoEntryContent = B->getMutableContent(G);
876       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
877     }
878   }
879 
880   return Error::success();
881 }
882 
883 } // End namespace orc.
884 } // End namespace llvm.
885