xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp (revision a90b9d0159070121c221b966469c3e36d912bf82) 
1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10 
11 #include "llvm/BinaryFormat/ELF.h"
12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
14 #include "llvm/ExecutionEngine/JITLink/ppc64.h"
15 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
16 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
17 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
18 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
19 #include "llvm/Support/BinaryByteStream.h"
20 #include "llvm/Support/Debug.h"
21 #include <optional>
22 
23 #define DEBUG_TYPE "orc"
24 
25 using namespace llvm;
26 using namespace llvm::orc;
27 using namespace llvm::orc::shared;
28 
29 namespace {
30 
31 class DSOHandleMaterializationUnit : public MaterializationUnit {
32 public:
33   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
34                                const SymbolStringPtr &DSOHandleSymbol)
35       : MaterializationUnit(
36             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
37         ENP(ENP) {}
38 
39   StringRef getName() const override { return "DSOHandleMU"; }
40 
41   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
42     unsigned PointerSize;
43     llvm::endianness Endianness;
44     jitlink::Edge::Kind EdgeKind;
45     const auto &TT = ENP.getExecutionSession().getTargetTriple();
46 
47     switch (TT.getArch()) {
48     case Triple::x86_64:
49       PointerSize = 8;
50       Endianness = llvm::endianness::little;
51       EdgeKind = jitlink::x86_64::Pointer64;
52       break;
53     case Triple::aarch64:
54       PointerSize = 8;
55       Endianness = llvm::endianness::little;
56       EdgeKind = jitlink::aarch64::Pointer64;
57       break;
58     case Triple::ppc64:
59       PointerSize = 8;
60       Endianness = llvm::endianness::big;
61       EdgeKind = jitlink::ppc64::Pointer64;
62       break;
63     case Triple::ppc64le:
64       PointerSize = 8;
65       Endianness = llvm::endianness::little;
66       EdgeKind = jitlink::ppc64::Pointer64;
67       break;
68     default:
69       llvm_unreachable("Unrecognized architecture");
70     }
71 
72     // void *__dso_handle = &__dso_handle;
73     auto G = std::make_unique<jitlink::LinkGraph>(
74         "<DSOHandleMU>", TT, PointerSize, Endianness,
75         jitlink::getGenericEdgeKindName);
76     auto &DSOHandleSection =
77         G->createSection(".data.__dso_handle", MemProt::Read);
78     auto &DSOHandleBlock = G->createContentBlock(
79         DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(),
80         8, 0);
81     auto &DSOHandleSymbol = G->addDefinedSymbol(
82         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
83         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
84     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
85 
86     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
87   }
88 
89   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
90 
91 private:
92   static MaterializationUnit::Interface
93   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
94                                   const SymbolStringPtr &DSOHandleSymbol) {
95     SymbolFlagsMap SymbolFlags;
96     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
97     return MaterializationUnit::Interface(std::move(SymbolFlags),
98                                           DSOHandleSymbol);
99   }
100 
101   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
102     static const char Content[8] = {0};
103     assert(PointerSize <= sizeof Content);
104     return {Content, PointerSize};
105   }
106 
107   ELFNixPlatform &ENP;
108 };
109 
110 } // end anonymous namespace
111 
112 namespace llvm {
113 namespace orc {
114 
115 Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create(
116     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
117     JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime,
118     std::optional<SymbolAliasMap> RuntimeAliases) {
119 
120   // If the target is not supported then bail out immediately.
121   if (!supportedTarget(ES.getTargetTriple()))
122     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
123                                        ES.getTargetTriple().str(),
124                                    inconvertibleErrorCode());
125 
126   auto &EPC = ES.getExecutorProcessControl();
127 
128   // Create default aliases if the caller didn't supply any.
129   if (!RuntimeAliases) {
130     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
131     if (!StandardRuntimeAliases)
132       return StandardRuntimeAliases.takeError();
133     RuntimeAliases = std::move(*StandardRuntimeAliases);
134   }
135 
136   // Define the aliases.
137   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
138     return std::move(Err);
139 
140   // Add JIT-dispatch function support symbols.
141   if (auto Err = PlatformJD.define(
142           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
143                             {EPC.getJITDispatchInfo().JITDispatchFunction,
144                              JITSymbolFlags::Exported}},
145                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
146                             {EPC.getJITDispatchInfo().JITDispatchContext,
147                              JITSymbolFlags::Exported}}})))
148     return std::move(Err);
149 
150   // Create the instance.
151   Error Err = Error::success();
152   auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
153       ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
154   if (Err)
155     return std::move(Err);
156   return std::move(P);
157 }
158 
159 Expected<std::unique_ptr<ELFNixPlatform>>
160 ELFNixPlatform::Create(ExecutionSession &ES,
161                        ObjectLinkingLayer &ObjLinkingLayer,
162                        JITDylib &PlatformJD, const char *OrcRuntimePath,
163                        std::optional<SymbolAliasMap> RuntimeAliases) {
164 
165   // Create a generator for the ORC runtime archive.
166   auto OrcRuntimeArchiveGenerator =
167       StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
168   if (!OrcRuntimeArchiveGenerator)
169     return OrcRuntimeArchiveGenerator.takeError();
170 
171   return Create(ES, ObjLinkingLayer, PlatformJD,
172                 std::move(*OrcRuntimeArchiveGenerator),
173                 std::move(RuntimeAliases));
174 }
175 
176 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
177   return JD.define(
178       std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
179 }
180 
181 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
182   return Error::success();
183 }
184 
185 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
186                                    const MaterializationUnit &MU) {
187   auto &JD = RT.getJITDylib();
188   const auto &InitSym = MU.getInitializerSymbol();
189   if (!InitSym)
190     return Error::success();
191 
192   RegisteredInitSymbols[&JD].add(InitSym,
193                                  SymbolLookupFlags::WeaklyReferencedSymbol);
194   LLVM_DEBUG({
195     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
196            << " for MU " << MU.getName() << "\n";
197   });
198   return Error::success();
199 }
200 
201 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
202   llvm_unreachable("Not supported yet");
203 }
204 
205 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
206                        ArrayRef<std::pair<const char *, const char *>> AL) {
207   for (auto &KV : AL) {
208     auto AliasName = ES.intern(KV.first);
209     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
210     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
211                                      JITSymbolFlags::Exported};
212   }
213 }
214 
215 Expected<SymbolAliasMap>
216 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
217                                         JITDylib &PlatformJD) {
218   SymbolAliasMap Aliases;
219   addAliases(ES, Aliases, requiredCXXAliases());
220   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
221   return Aliases;
222 }
223 
224 ArrayRef<std::pair<const char *, const char *>>
225 ELFNixPlatform::requiredCXXAliases() {
226   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
227       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
228       {"atexit", "__orc_rt_elfnix_atexit"}};
229 
230   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
231 }
232 
233 ArrayRef<std::pair<const char *, const char *>>
234 ELFNixPlatform::standardRuntimeUtilityAliases() {
235   static const std::pair<const char *, const char *>
236       StandardRuntimeUtilityAliases[] = {
237           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
238           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
239           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
240           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
241           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
242           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
243 
244   return ArrayRef<std::pair<const char *, const char *>>(
245       StandardRuntimeUtilityAliases);
246 }
247 
248 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
249   switch (TT.getArch()) {
250   case Triple::x86_64:
251   case Triple::aarch64:
252   // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
253   // right now.
254   case Triple::ppc64le:
255     return true;
256   default:
257     return false;
258   }
259 }
260 
261 ELFNixPlatform::ELFNixPlatform(
262     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
263     JITDylib &PlatformJD,
264     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
265     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
266       DSOHandleSymbol(ES.intern("__dso_handle")) {
267   ErrorAsOutParameter _(&Err);
268 
269   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
270 
271   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
272 
273   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
274   // the platform now), so set it up.
275   if (auto E2 = setupJITDylib(PlatformJD)) {
276     Err = std::move(E2);
277     return;
278   }
279 
280   RegisteredInitSymbols[&PlatformJD].add(
281       DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
282 
283   // Associate wrapper function tags with JIT-side function implementations.
284   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
285     Err = std::move(E2);
286     return;
287   }
288 
289   // Lookup addresses of runtime functions callable by the platform,
290   // call the platform bootstrap function to initialize the platform-state
291   // object in the executor.
292   if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
293     Err = std::move(E2);
294     return;
295   }
296 }
297 
298 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
299   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
300 
301   using GetInitializersSPSSig =
302       SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
303   WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
304       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
305           this, &ELFNixPlatform::rt_getInitializers);
306 
307   using GetDeinitializersSPSSig =
308       SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
309   WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
310       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
311           this, &ELFNixPlatform::rt_getDeinitializers);
312 
313   using LookupSymbolSPSSig =
314       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
315   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
316       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
317                                               &ELFNixPlatform::rt_lookupSymbol);
318 
319   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
320 }
321 
322 void ELFNixPlatform::getInitializersBuildSequencePhase(
323     SendInitializerSequenceFn SendResult, JITDylib &JD,
324     std::vector<JITDylibSP> DFSLinkOrder) {
325   ELFNixJITDylibInitializerSequence FullInitSeq;
326   {
327     std::lock_guard<std::mutex> Lock(PlatformMutex);
328     for (auto &InitJD : reverse(DFSLinkOrder)) {
329       LLVM_DEBUG({
330         dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
331                << "\" to sequence\n";
332       });
333       auto ISItr = InitSeqs.find(InitJD.get());
334       if (ISItr != InitSeqs.end()) {
335         FullInitSeq.emplace_back(std::move(ISItr->second));
336         InitSeqs.erase(ISItr);
337       }
338     }
339   }
340 
341   SendResult(std::move(FullInitSeq));
342 }
343 
344 void ELFNixPlatform::getInitializersLookupPhase(
345     SendInitializerSequenceFn SendResult, JITDylib &JD) {
346 
347   auto DFSLinkOrder = JD.getDFSLinkOrder();
348   if (!DFSLinkOrder) {
349     SendResult(DFSLinkOrder.takeError());
350     return;
351   }
352 
353   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
354   ES.runSessionLocked([&]() {
355     for (auto &InitJD : *DFSLinkOrder) {
356       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
357       if (RISItr != RegisteredInitSymbols.end()) {
358         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
359         RegisteredInitSymbols.erase(RISItr);
360       }
361     }
362   });
363 
364   // If there are no further init symbols to look up then move on to the next
365   // phase.
366   if (NewInitSymbols.empty()) {
367     getInitializersBuildSequencePhase(std::move(SendResult), JD,
368                                       std::move(*DFSLinkOrder));
369     return;
370   }
371 
372   // Otherwise issue a lookup and re-run this phase when it completes.
373   lookupInitSymbolsAsync(
374       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
375         if (Err)
376           SendResult(std::move(Err));
377         else
378           getInitializersLookupPhase(std::move(SendResult), JD);
379       },
380       ES, std::move(NewInitSymbols));
381 }
382 
383 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
384                                         StringRef JDName) {
385   LLVM_DEBUG({
386     dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
387   });
388 
389   JITDylib *JD = ES.getJITDylibByName(JDName);
390   if (!JD) {
391     LLVM_DEBUG({
392       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
393     });
394     SendResult(make_error<StringError>("No JITDylib named " + JDName,
395                                        inconvertibleErrorCode()));
396     return;
397   }
398 
399   getInitializersLookupPhase(std::move(SendResult), *JD);
400 }
401 
402 void ELFNixPlatform::rt_getDeinitializers(
403     SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
404   LLVM_DEBUG({
405     dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n";
406   });
407 
408   JITDylib *JD = nullptr;
409 
410   {
411     std::lock_guard<std::mutex> Lock(PlatformMutex);
412     auto I = HandleAddrToJITDylib.find(Handle);
413     if (I != HandleAddrToJITDylib.end())
414       JD = I->second;
415   }
416 
417   if (!JD) {
418     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
419     SendResult(make_error<StringError>("No JITDylib associated with handle " +
420                                            formatv("{0:x}", Handle),
421                                        inconvertibleErrorCode()));
422     return;
423   }
424 
425   SendResult(ELFNixJITDylibDeinitializerSequence());
426 }
427 
428 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
429                                      ExecutorAddr Handle,
430                                      StringRef SymbolName) {
431   LLVM_DEBUG({
432     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
433   });
434 
435   JITDylib *JD = nullptr;
436 
437   {
438     std::lock_guard<std::mutex> Lock(PlatformMutex);
439     auto I = HandleAddrToJITDylib.find(Handle);
440     if (I != HandleAddrToJITDylib.end())
441       JD = I->second;
442   }
443 
444   if (!JD) {
445     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
446     SendResult(make_error<StringError>("No JITDylib associated with handle " +
447                                            formatv("{0:x}", Handle),
448                                        inconvertibleErrorCode()));
449     return;
450   }
451 
452   // Use functor class to work around XL build compiler issue on AIX.
453   class RtLookupNotifyComplete {
454   public:
455     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
456         : SendResult(std::move(SendResult)) {}
457     void operator()(Expected<SymbolMap> Result) {
458       if (Result) {
459         assert(Result->size() == 1 && "Unexpected result map count");
460         SendResult(Result->begin()->second.getAddress());
461       } else {
462         SendResult(Result.takeError());
463       }
464     }
465 
466   private:
467     SendSymbolAddressFn SendResult;
468   };
469 
470   ES.lookup(
471       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
472       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
473       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
474 }
475 
476 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
477 
478   std::pair<const char *, ExecutorAddr *> Symbols[] = {
479       {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
480       {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
481       {"__orc_rt_elfnix_register_object_sections",
482        &orc_rt_elfnix_register_object_sections},
483       {"__orc_rt_elfnix_create_pthread_key",
484        &orc_rt_elfnix_create_pthread_key}};
485 
486   SymbolLookupSet RuntimeSymbols;
487   std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
488   for (const auto &KV : Symbols) {
489     auto Name = ES.intern(KV.first);
490     RuntimeSymbols.add(Name);
491     AddrsToRecord.push_back({std::move(Name), KV.second});
492   }
493 
494   auto RuntimeSymbolAddrs = ES.lookup(
495       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
496   if (!RuntimeSymbolAddrs)
497     return RuntimeSymbolAddrs.takeError();
498 
499   for (const auto &KV : AddrsToRecord) {
500     auto &Name = KV.first;
501     assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
502     *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress();
503   }
504 
505   auto PJDDSOHandle = ES.lookup(
506       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
507   if (!PJDDSOHandle)
508     return PJDDSOHandle.takeError();
509 
510   if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
511           orc_rt_elfnix_platform_bootstrap,
512           PJDDSOHandle->getAddress().getValue()))
513     return Err;
514 
515   // FIXME: Ordering is fuzzy here. We're probably best off saying
516   // "behavior is undefined if code that uses the runtime is added before
517   // the platform constructor returns", then move all this to the constructor.
518   RuntimeBootstrapped = true;
519   std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
520   {
521     std::lock_guard<std::mutex> Lock(PlatformMutex);
522     DeferredPOSRs = std::move(BootstrapPOSRs);
523   }
524 
525   for (auto &D : DeferredPOSRs)
526     if (auto Err = registerPerObjectSections(D))
527       return Err;
528 
529   return Error::success();
530 }
531 
532 Error ELFNixPlatform::registerInitInfo(
533     JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
534 
535   std::unique_lock<std::mutex> Lock(PlatformMutex);
536 
537   ELFNixJITDylibInitializers *InitSeq = nullptr;
538   {
539     auto I = InitSeqs.find(&JD);
540     if (I == InitSeqs.end()) {
541       // If there's no init sequence entry yet then we need to look up the
542       // header symbol to force creation of one.
543       Lock.unlock();
544 
545       auto SearchOrder =
546           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
547       if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
548         return Err;
549 
550       Lock.lock();
551       I = InitSeqs.find(&JD);
552       assert(I != InitSeqs.end() &&
553              "Entry missing after header symbol lookup?");
554     }
555     InitSeq = &I->second;
556   }
557 
558   for (auto *Sec : InitSections) {
559     // FIXME: Avoid copy here.
560     jitlink::SectionRange R(*Sec);
561     InitSeq->InitSections[Sec->getName()].push_back(R.getRange());
562   }
563 
564   return Error::success();
565 }
566 
567 Error ELFNixPlatform::registerPerObjectSections(
568     const ELFPerObjectSectionsToRegister &POSR) {
569 
570   if (!orc_rt_elfnix_register_object_sections)
571     return make_error<StringError>("Attempting to register per-object "
572                                    "sections, but runtime support has not "
573                                    "been loaded yet",
574                                    inconvertibleErrorCode());
575 
576   Error ErrResult = Error::success();
577   if (auto Err = ES.callSPSWrapper<shared::SPSError(
578                      SPSELFPerObjectSectionsToRegister)>(
579           orc_rt_elfnix_register_object_sections, ErrResult, POSR))
580     return Err;
581   return ErrResult;
582 }
583 
584 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
585   if (!orc_rt_elfnix_create_pthread_key)
586     return make_error<StringError>(
587         "Attempting to create pthread key in target, but runtime support has "
588         "not been loaded yet",
589         inconvertibleErrorCode());
590 
591   Expected<uint64_t> Result(0);
592   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
593           orc_rt_elfnix_create_pthread_key, Result))
594     return std::move(Err);
595   return Result;
596 }
597 
598 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
599     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
600     jitlink::PassConfiguration &Config) {
601 
602   // If the initializer symbol is the __dso_handle symbol then just add
603   // the DSO handle support passes.
604   if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
605     addDSOHandleSupportPasses(MR, Config);
606     // The DSOHandle materialization unit doesn't require any other
607     // support, so we can bail out early.
608     return;
609   }
610 
611   // If the object contains initializers then add passes to record them.
612   if (MR.getInitializerSymbol())
613     addInitializerSupportPasses(MR, Config);
614 
615   // Add passes for eh-frame and TLV support.
616   addEHAndTLVSupportPasses(MR, Config);
617 }
618 
619 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
620 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
621     MaterializationResponsibility &MR) {
622   std::lock_guard<std::mutex> Lock(PluginMutex);
623   auto I = InitSymbolDeps.find(&MR);
624   if (I != InitSymbolDeps.end()) {
625     SyntheticSymbolDependenciesMap Result;
626     Result[MR.getInitializerSymbol()] = std::move(I->second);
627     InitSymbolDeps.erase(&MR);
628     return Result;
629   }
630   return SyntheticSymbolDependenciesMap();
631 }
632 
633 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
634     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
635 
636   /// Preserve init sections.
637   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
638     if (auto Err = preserveInitSections(G, MR))
639       return Err;
640     return Error::success();
641   });
642 
643   Config.PostFixupPasses.push_back(
644       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
645         return registerInitSections(G, JD);
646       });
647 }
648 
649 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
650     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
651 
652   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
653                                             jitlink::LinkGraph &G) -> Error {
654     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
655       return Sym->getName() == *MP.DSOHandleSymbol;
656     });
657     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
658     {
659       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
660       auto HandleAddr = (*I)->getAddress();
661       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
662       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
663       MP.InitSeqs.insert(std::make_pair(
664           &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
665     }
666     return Error::success();
667   });
668 }
669 
670 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
671     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
672 
673   // Insert TLV lowering at the start of the PostPrunePasses, since we want
674   // it to run before GOT/PLT lowering.
675 
676   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
677   // pass has done. Because the TLS descriptor need to be allocate in GOT.
678   Config.PostPrunePasses.push_back(
679       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
680         return fixTLVSectionsAndEdges(G, JD);
681       });
682 
683   // Add a pass to register the final addresses of the eh-frame and TLV sections
684   // with the runtime.
685   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
686     ELFPerObjectSectionsToRegister POSR;
687 
688     if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
689       jitlink::SectionRange R(*EHFrameSection);
690       if (!R.empty())
691         POSR.EHFrameSection = R.getRange();
692     }
693 
694     // Get a pointer to the thread data section if there is one. It will be used
695     // below.
696     jitlink::Section *ThreadDataSection =
697         G.findSectionByName(ELFThreadDataSectionName);
698 
699     // Handle thread BSS section if there is one.
700     if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {
701       // If there's already a thread data section in this graph then merge the
702       // thread BSS section content into it, otherwise just treat the thread
703       // BSS section as the thread data section.
704       if (ThreadDataSection)
705         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
706       else
707         ThreadDataSection = ThreadBSSSection;
708     }
709 
710     // Having merged thread BSS (if present) and thread data (if present),
711     // record the resulting section range.
712     if (ThreadDataSection) {
713       jitlink::SectionRange R(*ThreadDataSection);
714       if (!R.empty())
715         POSR.ThreadDataSection = R.getRange();
716     }
717 
718     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
719 
720       // If we're still bootstrapping the runtime then just record this
721       // frame for now.
722       if (!MP.RuntimeBootstrapped) {
723         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
724         MP.BootstrapPOSRs.push_back(POSR);
725         return Error::success();
726       }
727 
728       // Otherwise register it immediately.
729       if (auto Err = MP.registerPerObjectSections(POSR))
730         return Err;
731     }
732 
733     return Error::success();
734   });
735 }
736 
737 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
738     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
739 
740   JITLinkSymbolSet InitSectionSymbols;
741   for (auto &InitSection : G.sections()) {
742     // Skip non-init sections.
743     if (!isELFInitializerSection(InitSection.getName()))
744       continue;
745 
746     // Make a pass over live symbols in the section: those blocks are already
747     // preserved.
748     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
749     for (auto &Sym : InitSection.symbols()) {
750       auto &B = Sym->getBlock();
751       if (Sym->isLive() && Sym->getOffset() == 0 &&
752           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
753         InitSectionSymbols.insert(Sym);
754         AlreadyLiveBlocks.insert(&B);
755       }
756     }
757 
758     // Add anonymous symbols to preserve any not-already-preserved blocks.
759     for (auto *B : InitSection.blocks())
760       if (!AlreadyLiveBlocks.count(B))
761         InitSectionSymbols.insert(
762             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
763   }
764 
765   if (!InitSectionSymbols.empty()) {
766     std::lock_guard<std::mutex> Lock(PluginMutex);
767     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
768   }
769 
770   return Error::success();
771 }
772 
773 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
774     jitlink::LinkGraph &G, JITDylib &JD) {
775 
776   SmallVector<jitlink::Section *> InitSections;
777 
778   LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
779 
780   for (auto &Sec : G.sections()) {
781     if (isELFInitializerSection(Sec.getName())) {
782       InitSections.push_back(&Sec);
783     }
784   }
785 
786   // Dump the scraped inits.
787   LLVM_DEBUG({
788     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
789     for (auto *Sec : InitSections) {
790       jitlink::SectionRange R(*Sec);
791       dbgs() << "  " << Sec->getName() << ": " << R.getRange() << "\n";
792     }
793   });
794 
795   return MP.registerInitInfo(JD, InitSections);
796 }
797 
798 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
799     jitlink::LinkGraph &G, JITDylib &JD) {
800 
801   for (auto *Sym : G.external_symbols()) {
802     if (Sym->getName() == "__tls_get_addr") {
803       Sym->setName("___orc_rt_elfnix_tls_get_addr");
804     } else if (Sym->getName() == "__tlsdesc_resolver") {
805       Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
806     }
807   }
808 
809   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
810 
811   if (TLSInfoEntrySection) {
812     std::optional<uint64_t> Key;
813     {
814       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
815       auto I = MP.JITDylibToPThreadKey.find(&JD);
816       if (I != MP.JITDylibToPThreadKey.end())
817         Key = I->second;
818     }
819     if (!Key) {
820       if (auto KeyOrErr = MP.createPThreadKey())
821         Key = *KeyOrErr;
822       else
823         return KeyOrErr.takeError();
824     }
825 
826     uint64_t PlatformKeyBits =
827         support::endian::byte_swap(*Key, G.getEndianness());
828 
829     for (auto *B : TLSInfoEntrySection->blocks()) {
830       // FIXME: The TLS descriptor byte length may different with different
831       // ISA
832       assert(B->getSize() == (G.getPointerSize() * 2) &&
833              "TLS descriptor must be 2 words length");
834       auto TLSInfoEntryContent = B->getMutableContent(G);
835       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
836     }
837   }
838 
839   return Error::success();
840 }
841 
842 } // End namespace orc.
843 } // End namespace llvm.
844