xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp (revision 258a0d760aa8b42899a000e30f610f900a402556) 
1 //===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10 
11 #include "llvm/BinaryFormat/ELF.h"
12 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
13 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
14 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
15 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
16 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
17 #include "llvm/Support/BinaryByteStream.h"
18 #include "llvm/Support/Debug.h"
19 #include <optional>
20 
21 #define DEBUG_TYPE "orc"
22 
23 using namespace llvm;
24 using namespace llvm::orc;
25 using namespace llvm::orc::shared;
26 
27 namespace {
28 
29 class DSOHandleMaterializationUnit : public MaterializationUnit {
30 public:
31   DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
32                                const SymbolStringPtr &DSOHandleSymbol)
33       : MaterializationUnit(
34             createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
35         ENP(ENP) {}
36 
37   StringRef getName() const override { return "DSOHandleMU"; }
38 
39   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
40     unsigned PointerSize;
41     support::endianness Endianness;
42     jitlink::Edge::Kind EdgeKind;
43     const auto &TT =
44         ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
45 
46     switch (TT.getArch()) {
47     case Triple::x86_64:
48       PointerSize = 8;
49       Endianness = support::endianness::little;
50       EdgeKind = jitlink::x86_64::Pointer64;
51       break;
52     case Triple::aarch64:
53       PointerSize = 8;
54       Endianness = support::endianness::little;
55       EdgeKind = jitlink::aarch64::Pointer64;
56       break;
57     default:
58       llvm_unreachable("Unrecognized architecture");
59     }
60 
61     // void *__dso_handle = &__dso_handle;
62     auto G = std::make_unique<jitlink::LinkGraph>(
63         "<DSOHandleMU>", TT, PointerSize, Endianness,
64         jitlink::getGenericEdgeKindName);
65     auto &DSOHandleSection =
66         G->createSection(".data.__dso_handle", MemProt::Read);
67     auto &DSOHandleBlock = G->createContentBlock(
68         DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(),
69         8, 0);
70     auto &DSOHandleSymbol = G->addDefinedSymbol(
71         DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
72         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
73     DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
74 
75     ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
76   }
77 
78   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
79 
80 private:
81   static MaterializationUnit::Interface
82   createDSOHandleSectionInterface(ELFNixPlatform &ENP,
83                                   const SymbolStringPtr &DSOHandleSymbol) {
84     SymbolFlagsMap SymbolFlags;
85     SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
86     return MaterializationUnit::Interface(std::move(SymbolFlags),
87                                           DSOHandleSymbol);
88   }
89 
90   ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
91     static const char Content[8] = {0};
92     assert(PointerSize <= sizeof Content);
93     return {Content, PointerSize};
94   }
95 
96   ELFNixPlatform &ENP;
97 };
98 
99 StringRef EHFrameSectionName = ".eh_frame";
100 StringRef InitArrayFuncSectionName = ".init_array";
101 
102 StringRef ThreadBSSSectionName = ".tbss";
103 StringRef ThreadDataSectionName = ".tdata";
104 
105 } // end anonymous namespace
106 
107 namespace llvm {
108 namespace orc {
109 
110 Expected<std::unique_ptr<ELFNixPlatform>>
111 ELFNixPlatform::Create(ExecutionSession &ES,
112                        ObjectLinkingLayer &ObjLinkingLayer,
113                        JITDylib &PlatformJD, const char *OrcRuntimePath,
114                        std::optional<SymbolAliasMap> RuntimeAliases) {
115 
116   auto &EPC = ES.getExecutorProcessControl();
117 
118   // If the target is not supported then bail out immediately.
119   if (!supportedTarget(EPC.getTargetTriple()))
120     return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
121                                        EPC.getTargetTriple().str(),
122                                    inconvertibleErrorCode());
123 
124   // Create default aliases if the caller didn't supply any.
125   if (!RuntimeAliases) {
126     auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
127     if (!StandardRuntimeAliases)
128       return StandardRuntimeAliases.takeError();
129     RuntimeAliases = std::move(*StandardRuntimeAliases);
130   }
131 
132   // Define the aliases.
133   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
134     return std::move(Err);
135 
136   // Add JIT-dispatch function support symbols.
137   if (auto Err = PlatformJD.define(absoluteSymbols(
138           {{ES.intern("__orc_rt_jit_dispatch"),
139             {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
140              JITSymbolFlags::Exported}},
141            {ES.intern("__orc_rt_jit_dispatch_ctx"),
142             {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
143              JITSymbolFlags::Exported}}})))
144     return std::move(Err);
145 
146   // Create a generator for the ORC runtime archive.
147   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load(
148       ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
149   if (!OrcRuntimeArchiveGenerator)
150     return OrcRuntimeArchiveGenerator.takeError();
151 
152   // Create the instance.
153   Error Err = Error::success();
154   auto P = std::unique_ptr<ELFNixPlatform>(
155       new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD,
156                          std::move(*OrcRuntimeArchiveGenerator), Err));
157   if (Err)
158     return std::move(Err);
159   return std::move(P);
160 }
161 
162 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
163   return JD.define(
164       std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol));
165 }
166 
167 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
168   return Error::success();
169 }
170 
171 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
172                                    const MaterializationUnit &MU) {
173   auto &JD = RT.getJITDylib();
174   const auto &InitSym = MU.getInitializerSymbol();
175   if (!InitSym)
176     return Error::success();
177 
178   RegisteredInitSymbols[&JD].add(InitSym,
179                                  SymbolLookupFlags::WeaklyReferencedSymbol);
180   LLVM_DEBUG({
181     dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
182            << " for MU " << MU.getName() << "\n";
183   });
184   return Error::success();
185 }
186 
187 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
188   llvm_unreachable("Not supported yet");
189 }
190 
191 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
192                        ArrayRef<std::pair<const char *, const char *>> AL) {
193   for (auto &KV : AL) {
194     auto AliasName = ES.intern(KV.first);
195     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
196     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
197                                      JITSymbolFlags::Exported};
198   }
199 }
200 
201 Expected<SymbolAliasMap>
202 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
203                                         JITDylib &PlatformJD) {
204   SymbolAliasMap Aliases;
205   addAliases(ES, Aliases, requiredCXXAliases());
206   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
207 
208   // Determine whether or not the libunwind extended-API function for
209   // dynamically registering an entire .eh_frame section is available.
210   // If it is not, we assume that libgcc_s is being used, and alias to
211   // its __register_frame with the same functionality.
212   auto RTRegisterFrame = ES.intern("__orc_rt_register_eh_frame_section");
213   auto LibUnwindRegisterFrame = ES.intern("__unw_add_dynamic_eh_frame_section");
214   auto RTDeregisterFrame = ES.intern("__orc_rt_deregister_eh_frame_section");
215   auto LibUnwindDeregisterFrame =
216       ES.intern("__unw_remove_dynamic_eh_frame_section");
217   auto SM = ES.lookup(makeJITDylibSearchOrder(&PlatformJD),
218                       SymbolLookupSet()
219                           .add(LibUnwindRegisterFrame,
220                                SymbolLookupFlags::WeaklyReferencedSymbol)
221                           .add(LibUnwindDeregisterFrame,
222                                SymbolLookupFlags::WeaklyReferencedSymbol));
223   if (!SM) { // Weak-ref means no "missing symbol" errors, so this must be
224              // something more serious that we should report.
225     return SM.takeError();
226   } else if (SM->size() == 2) {
227     LLVM_DEBUG({
228       dbgs() << "Using libunwind " << LibUnwindRegisterFrame
229              << " for unwind info registration\n";
230     });
231     Aliases[std::move(RTRegisterFrame)] = {LibUnwindRegisterFrame,
232                                            JITSymbolFlags::Exported};
233     Aliases[std::move(RTDeregisterFrame)] = {LibUnwindDeregisterFrame,
234                                              JITSymbolFlags::Exported};
235   } else {
236     // Since LLVM libunwind is not present, we assume that unwinding
237     // is provided by libgcc
238     LLVM_DEBUG({
239       dbgs() << "Using libgcc __register_frame"
240              << " for unwind info registration\n";
241     });
242     Aliases[std::move(RTRegisterFrame)] = {ES.intern("__register_frame"),
243                                            JITSymbolFlags::Exported};
244     Aliases[std::move(RTDeregisterFrame)] = {ES.intern("__deregister_frame"),
245                                              JITSymbolFlags::Exported};
246   }
247 
248   return Aliases;
249 }
250 
251 ArrayRef<std::pair<const char *, const char *>>
252 ELFNixPlatform::requiredCXXAliases() {
253   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
254       {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
255       {"atexit", "__orc_rt_elfnix_atexit"}};
256 
257   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
258 }
259 
260 ArrayRef<std::pair<const char *, const char *>>
261 ELFNixPlatform::standardRuntimeUtilityAliases() {
262   static const std::pair<const char *, const char *>
263       StandardRuntimeUtilityAliases[] = {
264           {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
265           {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
266           {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
267           {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
268           {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
269           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
270 
271   return ArrayRef<std::pair<const char *, const char *>>(
272       StandardRuntimeUtilityAliases);
273 }
274 
275 bool ELFNixPlatform::isInitializerSection(StringRef SecName) {
276   if (SecName.consume_front(InitArrayFuncSectionName) &&
277       (SecName.empty() || SecName[0] == '.'))
278     return true;
279   return false;
280 }
281 
282 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
283   switch (TT.getArch()) {
284   case Triple::x86_64:
285   case Triple::aarch64:
286     return true;
287   default:
288     return false;
289   }
290 }
291 
292 ELFNixPlatform::ELFNixPlatform(
293     ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer,
294     JITDylib &PlatformJD,
295     std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
296     : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
297       DSOHandleSymbol(ES.intern("__dso_handle")) {
298   ErrorAsOutParameter _(&Err);
299 
300   ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
301 
302   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
303 
304   // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
305   // the platform now), so set it up.
306   if (auto E2 = setupJITDylib(PlatformJD)) {
307     Err = std::move(E2);
308     return;
309   }
310 
311   RegisteredInitSymbols[&PlatformJD].add(
312       DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol);
313 
314   // Associate wrapper function tags with JIT-side function implementations.
315   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
316     Err = std::move(E2);
317     return;
318   }
319 
320   // Lookup addresses of runtime functions callable by the platform,
321   // call the platform bootstrap function to initialize the platform-state
322   // object in the executor.
323   if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) {
324     Err = std::move(E2);
325     return;
326   }
327 }
328 
329 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
330   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
331 
332   using GetInitializersSPSSig =
333       SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString);
334   WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] =
335       ES.wrapAsyncWithSPS<GetInitializersSPSSig>(
336           this, &ELFNixPlatform::rt_getInitializers);
337 
338   using GetDeinitializersSPSSig =
339       SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr);
340   WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] =
341       ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>(
342           this, &ELFNixPlatform::rt_getDeinitializers);
343 
344   using LookupSymbolSPSSig =
345       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
346   WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
347       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
348                                               &ELFNixPlatform::rt_lookupSymbol);
349 
350   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
351 }
352 
353 void ELFNixPlatform::getInitializersBuildSequencePhase(
354     SendInitializerSequenceFn SendResult, JITDylib &JD,
355     std::vector<JITDylibSP> DFSLinkOrder) {
356   ELFNixJITDylibInitializerSequence FullInitSeq;
357   {
358     std::lock_guard<std::mutex> Lock(PlatformMutex);
359     for (auto &InitJD : reverse(DFSLinkOrder)) {
360       LLVM_DEBUG({
361         dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName()
362                << "\" to sequence\n";
363       });
364       auto ISItr = InitSeqs.find(InitJD.get());
365       if (ISItr != InitSeqs.end()) {
366         FullInitSeq.emplace_back(std::move(ISItr->second));
367         InitSeqs.erase(ISItr);
368       }
369     }
370   }
371 
372   SendResult(std::move(FullInitSeq));
373 }
374 
375 void ELFNixPlatform::getInitializersLookupPhase(
376     SendInitializerSequenceFn SendResult, JITDylib &JD) {
377 
378   auto DFSLinkOrder = JD.getDFSLinkOrder();
379   if (!DFSLinkOrder) {
380     SendResult(DFSLinkOrder.takeError());
381     return;
382   }
383 
384   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
385   ES.runSessionLocked([&]() {
386     for (auto &InitJD : *DFSLinkOrder) {
387       auto RISItr = RegisteredInitSymbols.find(InitJD.get());
388       if (RISItr != RegisteredInitSymbols.end()) {
389         NewInitSymbols[InitJD.get()] = std::move(RISItr->second);
390         RegisteredInitSymbols.erase(RISItr);
391       }
392     }
393   });
394 
395   // If there are no further init symbols to look up then move on to the next
396   // phase.
397   if (NewInitSymbols.empty()) {
398     getInitializersBuildSequencePhase(std::move(SendResult), JD,
399                                       std::move(*DFSLinkOrder));
400     return;
401   }
402 
403   // Otherwise issue a lookup and re-run this phase when it completes.
404   lookupInitSymbolsAsync(
405       [this, SendResult = std::move(SendResult), &JD](Error Err) mutable {
406         if (Err)
407           SendResult(std::move(Err));
408         else
409           getInitializersLookupPhase(std::move(SendResult), JD);
410       },
411       ES, std::move(NewInitSymbols));
412 }
413 
414 void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult,
415                                         StringRef JDName) {
416   LLVM_DEBUG({
417     dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n";
418   });
419 
420   JITDylib *JD = ES.getJITDylibByName(JDName);
421   if (!JD) {
422     LLVM_DEBUG({
423       dbgs() << "  No such JITDylib \"" << JDName << "\". Sending error.\n";
424     });
425     SendResult(make_error<StringError>("No JITDylib named " + JDName,
426                                        inconvertibleErrorCode()));
427     return;
428   }
429 
430   getInitializersLookupPhase(std::move(SendResult), *JD);
431 }
432 
433 void ELFNixPlatform::rt_getDeinitializers(
434     SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) {
435   LLVM_DEBUG({
436     dbgs() << "ELFNixPlatform::rt_getDeinitializers(\""
437            << formatv("{0:x}", Handle.getValue()) << "\")\n";
438   });
439 
440   JITDylib *JD = nullptr;
441 
442   {
443     std::lock_guard<std::mutex> Lock(PlatformMutex);
444     auto I = HandleAddrToJITDylib.find(Handle);
445     if (I != HandleAddrToJITDylib.end())
446       JD = I->second;
447   }
448 
449   if (!JD) {
450     LLVM_DEBUG({
451       dbgs() << "  No JITDylib for handle "
452              << formatv("{0:x}", Handle.getValue()) << "\n";
453     });
454     SendResult(make_error<StringError>("No JITDylib associated with handle " +
455                                            formatv("{0:x}", Handle.getValue()),
456                                        inconvertibleErrorCode()));
457     return;
458   }
459 
460   SendResult(ELFNixJITDylibDeinitializerSequence());
461 }
462 
463 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
464                                      ExecutorAddr Handle,
465                                      StringRef SymbolName) {
466   LLVM_DEBUG({
467     dbgs() << "ELFNixPlatform::rt_lookupSymbol(\""
468            << formatv("{0:x}", Handle.getValue()) << "\")\n";
469   });
470 
471   JITDylib *JD = nullptr;
472 
473   {
474     std::lock_guard<std::mutex> Lock(PlatformMutex);
475     auto I = HandleAddrToJITDylib.find(Handle);
476     if (I != HandleAddrToJITDylib.end())
477       JD = I->second;
478   }
479 
480   if (!JD) {
481     LLVM_DEBUG({
482       dbgs() << "  No JITDylib for handle "
483              << formatv("{0:x}", Handle.getValue()) << "\n";
484     });
485     SendResult(make_error<StringError>("No JITDylib associated with handle " +
486                                            formatv("{0:x}", Handle.getValue()),
487                                        inconvertibleErrorCode()));
488     return;
489   }
490 
491   // Use functor class to work around XL build compiler issue on AIX.
492   class RtLookupNotifyComplete {
493   public:
494     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
495         : SendResult(std::move(SendResult)) {}
496     void operator()(Expected<SymbolMap> Result) {
497       if (Result) {
498         assert(Result->size() == 1 && "Unexpected result map count");
499         SendResult(ExecutorAddr(Result->begin()->second.getAddress()));
500       } else {
501         SendResult(Result.takeError());
502       }
503     }
504 
505   private:
506     SendSymbolAddressFn SendResult;
507   };
508 
509   ES.lookup(
510       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
511       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
512       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
513 }
514 
515 Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) {
516 
517   std::pair<const char *, ExecutorAddr *> Symbols[] = {
518       {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap},
519       {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown},
520       {"__orc_rt_elfnix_register_object_sections",
521        &orc_rt_elfnix_register_object_sections},
522       {"__orc_rt_elfnix_create_pthread_key",
523        &orc_rt_elfnix_create_pthread_key}};
524 
525   SymbolLookupSet RuntimeSymbols;
526   std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord;
527   for (const auto &KV : Symbols) {
528     auto Name = ES.intern(KV.first);
529     RuntimeSymbols.add(Name);
530     AddrsToRecord.push_back({std::move(Name), KV.second});
531   }
532 
533   auto RuntimeSymbolAddrs = ES.lookup(
534       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols);
535   if (!RuntimeSymbolAddrs)
536     return RuntimeSymbolAddrs.takeError();
537 
538   for (const auto &KV : AddrsToRecord) {
539     auto &Name = KV.first;
540     assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?");
541     KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress());
542   }
543 
544   auto PJDDSOHandle = ES.lookup(
545       {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol);
546   if (!PJDDSOHandle)
547     return PJDDSOHandle.takeError();
548 
549   if (auto Err = ES.callSPSWrapper<void(uint64_t)>(
550           orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress()))
551     return Err;
552 
553   // FIXME: Ordering is fuzzy here. We're probably best off saying
554   // "behavior is undefined if code that uses the runtime is added before
555   // the platform constructor returns", then move all this to the constructor.
556   RuntimeBootstrapped = true;
557   std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs;
558   {
559     std::lock_guard<std::mutex> Lock(PlatformMutex);
560     DeferredPOSRs = std::move(BootstrapPOSRs);
561   }
562 
563   for (auto &D : DeferredPOSRs)
564     if (auto Err = registerPerObjectSections(D))
565       return Err;
566 
567   return Error::success();
568 }
569 
570 Error ELFNixPlatform::registerInitInfo(
571     JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) {
572 
573   std::unique_lock<std::mutex> Lock(PlatformMutex);
574 
575   ELFNixJITDylibInitializers *InitSeq = nullptr;
576   {
577     auto I = InitSeqs.find(&JD);
578     if (I == InitSeqs.end()) {
579       // If there's no init sequence entry yet then we need to look up the
580       // header symbol to force creation of one.
581       Lock.unlock();
582 
583       auto SearchOrder =
584           JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; });
585       if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError())
586         return Err;
587 
588       Lock.lock();
589       I = InitSeqs.find(&JD);
590       assert(I != InitSeqs.end() &&
591              "Entry missing after header symbol lookup?");
592     }
593     InitSeq = &I->second;
594   }
595 
596   for (auto *Sec : InitSections) {
597     // FIXME: Avoid copy here.
598     jitlink::SectionRange R(*Sec);
599     InitSeq->InitSections[Sec->getName()].push_back(
600         {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())});
601   }
602 
603   return Error::success();
604 }
605 
606 Error ELFNixPlatform::registerPerObjectSections(
607     const ELFPerObjectSectionsToRegister &POSR) {
608 
609   if (!orc_rt_elfnix_register_object_sections)
610     return make_error<StringError>("Attempting to register per-object "
611                                    "sections, but runtime support has not "
612                                    "been loaded yet",
613                                    inconvertibleErrorCode());
614 
615   Error ErrResult = Error::success();
616   if (auto Err = ES.callSPSWrapper<shared::SPSError(
617                      SPSELFPerObjectSectionsToRegister)>(
618           orc_rt_elfnix_register_object_sections, ErrResult, POSR))
619     return Err;
620   return ErrResult;
621 }
622 
623 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
624   if (!orc_rt_elfnix_create_pthread_key)
625     return make_error<StringError>(
626         "Attempting to create pthread key in target, but runtime support has "
627         "not been loaded yet",
628         inconvertibleErrorCode());
629 
630   Expected<uint64_t> Result(0);
631   if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
632           orc_rt_elfnix_create_pthread_key, Result))
633     return std::move(Err);
634   return Result;
635 }
636 
637 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
638     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
639     jitlink::PassConfiguration &Config) {
640 
641   // If the initializer symbol is the __dso_handle symbol then just add
642   // the DSO handle support passes.
643   if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) {
644     addDSOHandleSupportPasses(MR, Config);
645     // The DSOHandle materialization unit doesn't require any other
646     // support, so we can bail out early.
647     return;
648   }
649 
650   // If the object contains initializers then add passes to record them.
651   if (MR.getInitializerSymbol())
652     addInitializerSupportPasses(MR, Config);
653 
654   // Add passes for eh-frame and TLV support.
655   addEHAndTLVSupportPasses(MR, Config);
656 }
657 
658 ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap
659 ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies(
660     MaterializationResponsibility &MR) {
661   std::lock_guard<std::mutex> Lock(PluginMutex);
662   auto I = InitSymbolDeps.find(&MR);
663   if (I != InitSymbolDeps.end()) {
664     SyntheticSymbolDependenciesMap Result;
665     Result[MR.getInitializerSymbol()] = std::move(I->second);
666     InitSymbolDeps.erase(&MR);
667     return Result;
668   }
669   return SyntheticSymbolDependenciesMap();
670 }
671 
672 void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses(
673     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
674 
675   /// Preserve init sections.
676   Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error {
677     if (auto Err = preserveInitSections(G, MR))
678       return Err;
679     return Error::success();
680   });
681 
682   Config.PostFixupPasses.push_back(
683       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
684         return registerInitSections(G, JD);
685       });
686 }
687 
688 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
689     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
690 
691   Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
692                                             jitlink::LinkGraph &G) -> Error {
693     auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
694       return Sym->getName() == *MP.DSOHandleSymbol;
695     });
696     assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
697     {
698       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
699       auto HandleAddr = (*I)->getAddress();
700       MP.HandleAddrToJITDylib[HandleAddr] = &JD;
701       assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists");
702       MP.InitSeqs.insert(std::make_pair(
703           &JD, ELFNixJITDylibInitializers(JD.getName(), HandleAddr)));
704     }
705     return Error::success();
706   });
707 }
708 
709 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
710     MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
711 
712   // Insert TLV lowering at the start of the PostPrunePasses, since we want
713   // it to run before GOT/PLT lowering.
714 
715   // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
716   // pass has done. Because the TLS descriptor need to be allocate in GOT.
717   Config.PostPrunePasses.push_back(
718       [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
719         return fixTLVSectionsAndEdges(G, JD);
720       });
721 
722   // Add a pass to register the final addresses of the eh-frame and TLV sections
723   // with the runtime.
724   Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error {
725     ELFPerObjectSectionsToRegister POSR;
726 
727     if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) {
728       jitlink::SectionRange R(*EHFrameSection);
729       if (!R.empty())
730         POSR.EHFrameSection = {ExecutorAddr(R.getStart()),
731                                ExecutorAddr(R.getEnd())};
732     }
733 
734     // Get a pointer to the thread data section if there is one. It will be used
735     // below.
736     jitlink::Section *ThreadDataSection =
737         G.findSectionByName(ThreadDataSectionName);
738 
739     // Handle thread BSS section if there is one.
740     if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) {
741       // If there's already a thread data section in this graph then merge the
742       // thread BSS section content into it, otherwise just treat the thread
743       // BSS section as the thread data section.
744       if (ThreadDataSection)
745         G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
746       else
747         ThreadDataSection = ThreadBSSSection;
748     }
749 
750     // Having merged thread BSS (if present) and thread data (if present),
751     // record the resulting section range.
752     if (ThreadDataSection) {
753       jitlink::SectionRange R(*ThreadDataSection);
754       if (!R.empty())
755         POSR.ThreadDataSection = {ExecutorAddr(R.getStart()),
756                                   ExecutorAddr(R.getEnd())};
757     }
758 
759     if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
760 
761       // If we're still bootstrapping the runtime then just record this
762       // frame for now.
763       if (!MP.RuntimeBootstrapped) {
764         std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
765         MP.BootstrapPOSRs.push_back(POSR);
766         return Error::success();
767       }
768 
769       // Otherwise register it immediately.
770       if (auto Err = MP.registerPerObjectSections(POSR))
771         return Err;
772     }
773 
774     return Error::success();
775   });
776 }
777 
778 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
779     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
780 
781   JITLinkSymbolSet InitSectionSymbols;
782   for (auto &InitSection : G.sections()) {
783     // Skip non-init sections.
784     if (!isInitializerSection(InitSection.getName()))
785       continue;
786 
787     // Make a pass over live symbols in the section: those blocks are already
788     // preserved.
789     DenseSet<jitlink::Block *> AlreadyLiveBlocks;
790     for (auto &Sym : InitSection.symbols()) {
791       auto &B = Sym->getBlock();
792       if (Sym->isLive() && Sym->getOffset() == 0 &&
793           Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) {
794         InitSectionSymbols.insert(Sym);
795         AlreadyLiveBlocks.insert(&B);
796       }
797     }
798 
799     // Add anonymous symbols to preserve any not-already-preserved blocks.
800     for (auto *B : InitSection.blocks())
801       if (!AlreadyLiveBlocks.count(B))
802         InitSectionSymbols.insert(
803             &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true));
804   }
805 
806   if (!InitSectionSymbols.empty()) {
807     std::lock_guard<std::mutex> Lock(PluginMutex);
808     InitSymbolDeps[&MR] = std::move(InitSectionSymbols);
809   }
810 
811   return Error::success();
812 }
813 
814 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
815     jitlink::LinkGraph &G, JITDylib &JD) {
816 
817   SmallVector<jitlink::Section *> InitSections;
818 
819   LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; });
820 
821   for (auto &Sec : G.sections()) {
822     if (isInitializerSection(Sec.getName())) {
823       InitSections.push_back(&Sec);
824     }
825   }
826 
827   // Dump the scraped inits.
828   LLVM_DEBUG({
829     dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
830     for (auto *Sec : InitSections) {
831       jitlink::SectionRange R(*Sec);
832       dbgs() << "  " << Sec->getName() << ": "
833              << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n";
834     }
835   });
836 
837   return MP.registerInitInfo(JD, InitSections);
838 }
839 
840 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
841     jitlink::LinkGraph &G, JITDylib &JD) {
842 
843   for (auto *Sym : G.external_symbols()) {
844     if (Sym->getName() == "__tls_get_addr") {
845       Sym->setName("___orc_rt_elfnix_tls_get_addr");
846     } else if (Sym->getName() == "__tlsdesc_resolver") {
847       Sym->setName("___orc_rt_elfnix_tlsdesc_resolver");
848     }
849   }
850 
851   auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
852 
853   if (TLSInfoEntrySection) {
854     std::optional<uint64_t> Key;
855     {
856       std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
857       auto I = MP.JITDylibToPThreadKey.find(&JD);
858       if (I != MP.JITDylibToPThreadKey.end())
859         Key = I->second;
860     }
861     if (!Key) {
862       if (auto KeyOrErr = MP.createPThreadKey())
863         Key = *KeyOrErr;
864       else
865         return KeyOrErr.takeError();
866     }
867 
868     uint64_t PlatformKeyBits =
869         support::endian::byte_swap(*Key, G.getEndianness());
870 
871     for (auto *B : TLSInfoEntrySection->blocks()) {
872       // FIXME: The TLS descriptor byte length may different with different
873       // ISA
874       assert(B->getSize() == (G.getPointerSize() * 2) &&
875              "TLS descriptor must be 2 words length");
876       auto TLSInfoEntryContent = B->getMutableContent(G);
877       memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
878     }
879   }
880 
881   return Error::success();
882 }
883 
884 } // End namespace orc.
885 } // End namespace llvm.
886