1 //===----- ELFNixPlatform.cpp - Utilities for executing ELFNix in Orc -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h"
10
11 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
12 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
13 #include "llvm/ExecutionEngine/JITLink/ppc64.h"
14 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
15 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
16 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
17 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
18 #include "llvm/Support/Debug.h"
19 #include <optional>
20
21 #define DEBUG_TYPE "orc"
22
23 using namespace llvm;
24 using namespace llvm::orc;
25 using namespace llvm::orc::shared;
26
27 namespace {
28
29 template <typename SPSSerializer, typename... ArgTs>
30 shared::WrapperFunctionCall::ArgDataBufferType
getArgDataBufferType(const ArgTs &...Args)31 getArgDataBufferType(const ArgTs &...Args) {
32 shared::WrapperFunctionCall::ArgDataBufferType ArgData;
33 ArgData.resize(SPSSerializer::size(Args...));
34 SPSOutputBuffer OB(ArgData.empty() ? nullptr : ArgData.data(),
35 ArgData.size());
36 if (SPSSerializer::serialize(OB, Args...))
37 return ArgData;
38 return {};
39 }
40
createPlatformGraph(ELFNixPlatform & MOP,std::string Name)41 std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(ELFNixPlatform &MOP,
42 std::string Name) {
43 auto &ES = MOP.getExecutionSession();
44 return std::make_unique<jitlink::LinkGraph>(
45 std::move(Name), ES.getSymbolStringPool(), ES.getTargetTriple(),
46 SubtargetFeatures(), jitlink::getGenericEdgeKindName);
47 }
48
49 // Creates a Bootstrap-Complete LinkGraph to run deferred actions.
50 class ELFNixPlatformCompleteBootstrapMaterializationUnit
51 : public MaterializationUnit {
52 public:
ELFNixPlatformCompleteBootstrapMaterializationUnit(ELFNixPlatform & MOP,StringRef PlatformJDName,SymbolStringPtr CompleteBootstrapSymbol,DeferredRuntimeFnMap DeferredAAs,ExecutorAddr ELFNixHeaderAddr,ExecutorAddr PlatformBootstrap,ExecutorAddr PlatformShutdown,ExecutorAddr RegisterJITDylib,ExecutorAddr DeregisterJITDylib)53 ELFNixPlatformCompleteBootstrapMaterializationUnit(
54 ELFNixPlatform &MOP, StringRef PlatformJDName,
55 SymbolStringPtr CompleteBootstrapSymbol, DeferredRuntimeFnMap DeferredAAs,
56 ExecutorAddr ELFNixHeaderAddr, ExecutorAddr PlatformBootstrap,
57 ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib,
58 ExecutorAddr DeregisterJITDylib)
59 : MaterializationUnit(
60 {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}),
61 MOP(MOP), PlatformJDName(PlatformJDName),
62 CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)),
63 DeferredAAsMap(std::move(DeferredAAs)),
64 ELFNixHeaderAddr(ELFNixHeaderAddr),
65 PlatformBootstrap(PlatformBootstrap),
66 PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib),
67 DeregisterJITDylib(DeregisterJITDylib) {}
68
getName() const69 StringRef getName() const override {
70 return "ELFNixPlatformCompleteBootstrap";
71 }
72
materialize(std::unique_ptr<MaterializationResponsibility> R)73 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
74 using namespace jitlink;
75 auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>");
76 auto &PlaceholderSection =
77 G->createSection("__orc_rt_cplt_bs", MemProt::Read);
78 auto &PlaceholderBlock =
79 G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0);
80 G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1,
81 Linkage::Strong, Scope::Hidden, false, true);
82
83 // 1. Bootstrap the platform support code.
84 G->allocActions().push_back(
85 {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
86 PlatformBootstrap, ELFNixHeaderAddr)),
87 cantFail(
88 WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))});
89
90 // 2. Register the platform JITDylib.
91 G->allocActions().push_back(
92 {cantFail(WrapperFunctionCall::Create<
93 SPSArgList<SPSString, SPSExecutorAddr>>(
94 RegisterJITDylib, PlatformJDName, ELFNixHeaderAddr)),
95 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
96 DeregisterJITDylib, ELFNixHeaderAddr))});
97
98 // 4. Add the deferred actions to the graph.
99 for (auto &[Fn, CallDatas] : DeferredAAsMap) {
100 for (auto &CallData : CallDatas) {
101 G->allocActions().push_back(
102 {WrapperFunctionCall(Fn.first->Addr, std::move(CallData.first)),
103 WrapperFunctionCall(Fn.second->Addr, std::move(CallData.second))});
104 }
105 }
106
107 MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
108 }
109
discard(const JITDylib & JD,const SymbolStringPtr & Sym)110 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
111
112 private:
113 ELFNixPlatform &MOP;
114 StringRef PlatformJDName;
115 SymbolStringPtr CompleteBootstrapSymbol;
116 DeferredRuntimeFnMap DeferredAAsMap;
117 ExecutorAddr ELFNixHeaderAddr;
118 ExecutorAddr PlatformBootstrap;
119 ExecutorAddr PlatformShutdown;
120 ExecutorAddr RegisterJITDylib;
121 ExecutorAddr DeregisterJITDylib;
122 };
123
124 class DSOHandleMaterializationUnit : public MaterializationUnit {
125 public:
DSOHandleMaterializationUnit(ELFNixPlatform & ENP,const SymbolStringPtr & DSOHandleSymbol)126 DSOHandleMaterializationUnit(ELFNixPlatform &ENP,
127 const SymbolStringPtr &DSOHandleSymbol)
128 : MaterializationUnit(
129 createDSOHandleSectionInterface(ENP, DSOHandleSymbol)),
130 ENP(ENP) {}
131
getName() const132 StringRef getName() const override { return "DSOHandleMU"; }
133
materialize(std::unique_ptr<MaterializationResponsibility> R)134 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
135
136 auto &ES = ENP.getExecutionSession();
137
138 jitlink::Edge::Kind EdgeKind;
139
140 switch (ES.getTargetTriple().getArch()) {
141 case Triple::x86_64:
142 EdgeKind = jitlink::x86_64::Pointer64;
143 break;
144 case Triple::aarch64:
145 EdgeKind = jitlink::aarch64::Pointer64;
146 break;
147 case Triple::ppc64:
148 EdgeKind = jitlink::ppc64::Pointer64;
149 break;
150 case Triple::ppc64le:
151 EdgeKind = jitlink::ppc64::Pointer64;
152 break;
153 case Triple::loongarch64:
154 EdgeKind = jitlink::loongarch::Pointer64;
155 break;
156 default:
157 llvm_unreachable("Unrecognized architecture");
158 }
159
160 // void *__dso_handle = &__dso_handle;
161 auto G = std::make_unique<jitlink::LinkGraph>(
162 "<DSOHandleMU>", ES.getSymbolStringPool(), ES.getTargetTriple(),
163 SubtargetFeatures(), jitlink::getGenericEdgeKindName);
164 auto &DSOHandleSection =
165 G->createSection(".data.__dso_handle", MemProt::Read);
166 auto &DSOHandleBlock = G->createContentBlock(
167 DSOHandleSection, getDSOHandleContent(G->getPointerSize()),
168 orc::ExecutorAddr(), 8, 0);
169 auto &DSOHandleSymbol = G->addDefinedSymbol(
170 DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(),
171 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
172 DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0);
173
174 ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
175 }
176
discard(const JITDylib & JD,const SymbolStringPtr & Sym)177 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
178
179 private:
180 static MaterializationUnit::Interface
createDSOHandleSectionInterface(ELFNixPlatform & ENP,const SymbolStringPtr & DSOHandleSymbol)181 createDSOHandleSectionInterface(ELFNixPlatform &ENP,
182 const SymbolStringPtr &DSOHandleSymbol) {
183 SymbolFlagsMap SymbolFlags;
184 SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported;
185 return MaterializationUnit::Interface(std::move(SymbolFlags),
186 DSOHandleSymbol);
187 }
188
getDSOHandleContent(size_t PointerSize)189 ArrayRef<char> getDSOHandleContent(size_t PointerSize) {
190 static const char Content[8] = {0};
191 assert(PointerSize <= sizeof Content);
192 return {Content, PointerSize};
193 }
194
195 ELFNixPlatform &ENP;
196 };
197
198 } // end anonymous namespace
199
200 namespace llvm {
201 namespace orc {
202
203 Expected<std::unique_ptr<ELFNixPlatform>>
Create(ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<DefinitionGenerator> OrcRuntime,std::optional<SymbolAliasMap> RuntimeAliases)204 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
205 JITDylib &PlatformJD,
206 std::unique_ptr<DefinitionGenerator> OrcRuntime,
207 std::optional<SymbolAliasMap> RuntimeAliases) {
208
209 auto &ES = ObjLinkingLayer.getExecutionSession();
210
211 // If the target is not supported then bail out immediately.
212 if (!supportedTarget(ES.getTargetTriple()))
213 return make_error<StringError>("Unsupported ELFNixPlatform triple: " +
214 ES.getTargetTriple().str(),
215 inconvertibleErrorCode());
216
217 auto &EPC = ES.getExecutorProcessControl();
218
219 // Create default aliases if the caller didn't supply any.
220 if (!RuntimeAliases) {
221 auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD);
222 if (!StandardRuntimeAliases)
223 return StandardRuntimeAliases.takeError();
224 RuntimeAliases = std::move(*StandardRuntimeAliases);
225 }
226
227 // Define the aliases.
228 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
229 return std::move(Err);
230
231 // Add JIT-dispatch function support symbols.
232 if (auto Err = PlatformJD.define(
233 absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
234 {EPC.getJITDispatchInfo().JITDispatchFunction,
235 JITSymbolFlags::Exported}},
236 {ES.intern("__orc_rt_jit_dispatch_ctx"),
237 {EPC.getJITDispatchInfo().JITDispatchContext,
238 JITSymbolFlags::Exported}}})))
239 return std::move(Err);
240
241 // Create the instance.
242 Error Err = Error::success();
243 auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform(
244 ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err));
245 if (Err)
246 return std::move(Err);
247 return std::move(P);
248 }
249
250 Expected<std::unique_ptr<ELFNixPlatform>>
Create(ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,const char * OrcRuntimePath,std::optional<SymbolAliasMap> RuntimeAliases)251 ELFNixPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer,
252 JITDylib &PlatformJD, const char *OrcRuntimePath,
253 std::optional<SymbolAliasMap> RuntimeAliases) {
254
255 // Create a generator for the ORC runtime archive.
256 auto OrcRuntimeArchiveGenerator =
257 StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath);
258 if (!OrcRuntimeArchiveGenerator)
259 return OrcRuntimeArchiveGenerator.takeError();
260
261 return Create(ObjLinkingLayer, PlatformJD,
262 std::move(*OrcRuntimeArchiveGenerator),
263 std::move(RuntimeAliases));
264 }
265
setupJITDylib(JITDylib & JD)266 Error ELFNixPlatform::setupJITDylib(JITDylib &JD) {
267 if (auto Err = JD.define(std::make_unique<DSOHandleMaterializationUnit>(
268 *this, DSOHandleSymbol)))
269 return Err;
270
271 return ES.lookup({&JD}, DSOHandleSymbol).takeError();
272 }
273
teardownJITDylib(JITDylib & JD)274 Error ELFNixPlatform::teardownJITDylib(JITDylib &JD) {
275 std::lock_guard<std::mutex> Lock(PlatformMutex);
276 auto I = JITDylibToHandleAddr.find(&JD);
277 if (I != JITDylibToHandleAddr.end()) {
278 assert(HandleAddrToJITDylib.count(I->second) &&
279 "HandleAddrToJITDylib missing entry");
280 HandleAddrToJITDylib.erase(I->second);
281 JITDylibToHandleAddr.erase(I);
282 }
283 return Error::success();
284 }
285
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)286 Error ELFNixPlatform::notifyAdding(ResourceTracker &RT,
287 const MaterializationUnit &MU) {
288
289 auto &JD = RT.getJITDylib();
290 const auto &InitSym = MU.getInitializerSymbol();
291 if (!InitSym)
292 return Error::success();
293
294 RegisteredInitSymbols[&JD].add(InitSym,
295 SymbolLookupFlags::WeaklyReferencedSymbol);
296 LLVM_DEBUG({
297 dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym
298 << " for MU " << MU.getName() << "\n";
299 });
300 return Error::success();
301 }
302
notifyRemoving(ResourceTracker & RT)303 Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) {
304 llvm_unreachable("Not supported yet");
305 }
306
addAliases(ExecutionSession & ES,SymbolAliasMap & Aliases,ArrayRef<std::pair<const char *,const char * >> AL)307 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
308 ArrayRef<std::pair<const char *, const char *>> AL) {
309 for (auto &KV : AL) {
310 auto AliasName = ES.intern(KV.first);
311 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
312 Aliases[std::move(AliasName)] = {ES.intern(KV.second),
313 JITSymbolFlags::Exported};
314 }
315 }
316
317 Expected<SymbolAliasMap>
standardPlatformAliases(ExecutionSession & ES,JITDylib & PlatformJD)318 ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES,
319 JITDylib &PlatformJD) {
320 SymbolAliasMap Aliases;
321 addAliases(ES, Aliases, requiredCXXAliases());
322 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
323 addAliases(ES, Aliases, standardLazyCompilationAliases());
324 return Aliases;
325 }
326
327 ArrayRef<std::pair<const char *, const char *>>
requiredCXXAliases()328 ELFNixPlatform::requiredCXXAliases() {
329 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
330 {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"},
331 {"atexit", "__orc_rt_elfnix_atexit"}};
332
333 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
334 }
335
336 ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases()337 ELFNixPlatform::standardRuntimeUtilityAliases() {
338 static const std::pair<const char *, const char *>
339 StandardRuntimeUtilityAliases[] = {
340 {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"},
341 {"__orc_rt_jit_dlerror", "__orc_rt_elfnix_jit_dlerror"},
342 {"__orc_rt_jit_dlopen", "__orc_rt_elfnix_jit_dlopen"},
343 {"__orc_rt_jit_dlupdate", "__orc_rt_elfnix_jit_dlupdate"},
344 {"__orc_rt_jit_dlclose", "__orc_rt_elfnix_jit_dlclose"},
345 {"__orc_rt_jit_dlsym", "__orc_rt_elfnix_jit_dlsym"},
346 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
347
348 return ArrayRef<std::pair<const char *, const char *>>(
349 StandardRuntimeUtilityAliases);
350 }
351
352 ArrayRef<std::pair<const char *, const char *>>
standardLazyCompilationAliases()353 ELFNixPlatform::standardLazyCompilationAliases() {
354 static const std::pair<const char *, const char *>
355 StandardLazyCompilationAliases[] = {
356 {"__orc_rt_reenter", "__orc_rt_sysv_reenter"}};
357
358 return ArrayRef<std::pair<const char *, const char *>>(
359 StandardLazyCompilationAliases);
360 }
361
supportedTarget(const Triple & TT)362 bool ELFNixPlatform::supportedTarget(const Triple &TT) {
363 switch (TT.getArch()) {
364 case Triple::x86_64:
365 case Triple::aarch64:
366 // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported
367 // right now.
368 case Triple::ppc64le:
369 case Triple::loongarch64:
370 return true;
371 default:
372 return false;
373 }
374 }
375
ELFNixPlatform(ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,Error & Err)376 ELFNixPlatform::ELFNixPlatform(
377 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
378 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err)
379 : ES(ObjLinkingLayer.getExecutionSession()), PlatformJD(PlatformJD),
380 ObjLinkingLayer(ObjLinkingLayer),
381 DSOHandleSymbol(ES.intern("__dso_handle")) {
382 ErrorAsOutParameter _(Err);
383 ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this));
384
385 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
386
387 BootstrapInfo BI;
388 Bootstrap = &BI;
389
390 // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating
391 // the platform now), so set it up.
392 if (auto E2 = setupJITDylib(PlatformJD)) {
393 Err = std::move(E2);
394 return;
395 }
396
397 // Step (2) Request runtime registration functions to trigger
398 // materialization..
399 if ((Err = ES.lookup(
400 makeJITDylibSearchOrder(&PlatformJD),
401 SymbolLookupSet(
402 {PlatformBootstrap.Name, PlatformShutdown.Name,
403 RegisterJITDylib.Name, DeregisterJITDylib.Name,
404 RegisterInitSections.Name, DeregisterInitSections.Name,
405 RegisterObjectSections.Name,
406 DeregisterObjectSections.Name, CreatePThreadKey.Name}))
407 .takeError()))
408 return;
409
410 // Step (3) Wait for any incidental linker work to complete.
411 {
412 std::unique_lock<std::mutex> Lock(BI.Mutex);
413 BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; });
414 Bootstrap = nullptr;
415 }
416
417 // Step (4) Add complete-bootstrap materialization unit and request.
418 auto BootstrapCompleteSymbol =
419 ES.intern("__orc_rt_elfnix_complete_bootstrap");
420 if ((Err = PlatformJD.define(
421 std::make_unique<ELFNixPlatformCompleteBootstrapMaterializationUnit>(
422 *this, PlatformJD.getName(), BootstrapCompleteSymbol,
423 std::move(BI.DeferredRTFnMap), BI.ELFNixHeaderAddr,
424 PlatformBootstrap.Addr, PlatformShutdown.Addr,
425 RegisterJITDylib.Addr, DeregisterJITDylib.Addr))))
426 return;
427 if ((Err = ES.lookup(makeJITDylibSearchOrder(
428 &PlatformJD, JITDylibLookupFlags::MatchAllSymbols),
429 std::move(BootstrapCompleteSymbol))
430 .takeError()))
431 return;
432
433 // Associate wrapper function tags with JIT-side function implementations.
434 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
435 Err = std::move(E2);
436 return;
437 }
438 }
439
associateRuntimeSupportFunctions(JITDylib & PlatformJD)440 Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
441 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
442
443 using RecordInitializersSPSSig =
444 SPSExpected<SPSELFNixJITDylibDepInfoMap>(SPSExecutorAddr);
445 WFs[ES.intern("__orc_rt_elfnix_push_initializers_tag")] =
446 ES.wrapAsyncWithSPS<RecordInitializersSPSSig>(
447 this, &ELFNixPlatform::rt_recordInitializers);
448
449 using LookupSymbolSPSSig =
450 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
451 WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] =
452 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
453 &ELFNixPlatform::rt_lookupSymbol);
454
455 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
456 }
457
pushInitializersLoop(PushInitializersSendResultFn SendResult,JITDylibSP JD)458 void ELFNixPlatform::pushInitializersLoop(
459 PushInitializersSendResultFn SendResult, JITDylibSP JD) {
460 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
461 DenseMap<JITDylib *, SmallVector<JITDylib *>> JDDepMap;
462 SmallVector<JITDylib *, 16> Worklist({JD.get()});
463
464 ES.runSessionLocked([&]() {
465 while (!Worklist.empty()) {
466 // FIXME: Check for defunct dylibs.
467
468 auto DepJD = Worklist.back();
469 Worklist.pop_back();
470
471 // If we've already visited this JITDylib on this iteration then continue.
472 auto [It, Inserted] = JDDepMap.try_emplace(DepJD);
473 if (!Inserted)
474 continue;
475
476 // Add dep info.
477 auto &DM = It->second;
478 DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
479 for (auto &KV : O) {
480 if (KV.first == DepJD)
481 continue;
482 DM.push_back(KV.first);
483 Worklist.push_back(KV.first);
484 }
485 });
486
487 // Add any registered init symbols.
488 auto RISItr = RegisteredInitSymbols.find(DepJD);
489 if (RISItr != RegisteredInitSymbols.end()) {
490 NewInitSymbols[DepJD] = std::move(RISItr->second);
491 RegisteredInitSymbols.erase(RISItr);
492 }
493 }
494 });
495
496 // If there are no further init symbols to look up then send the link order
497 // (as a list of header addresses) to the caller.
498 if (NewInitSymbols.empty()) {
499
500 // To make the list intelligible to the runtime we need to convert all
501 // JITDylib pointers to their header addresses. Only include JITDylibs
502 // that appear in the JITDylibToHandleAddr map (i.e. those that have been
503 // through setupJITDylib) -- bare JITDylibs aren't managed by the platform.
504 DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs;
505 HeaderAddrs.reserve(JDDepMap.size());
506 {
507 std::lock_guard<std::mutex> Lock(PlatformMutex);
508 for (auto &KV : JDDepMap) {
509 auto I = JITDylibToHandleAddr.find(KV.first);
510 if (I != JITDylibToHandleAddr.end())
511 HeaderAddrs[KV.first] = I->second;
512 }
513 }
514
515 // Build the dep info map to return.
516 ELFNixJITDylibDepInfoMap DIM;
517 DIM.reserve(JDDepMap.size());
518 for (auto &KV : JDDepMap) {
519 auto HI = HeaderAddrs.find(KV.first);
520 // Skip unmanaged JITDylibs.
521 if (HI == HeaderAddrs.end())
522 continue;
523 auto H = HI->second;
524 ELFNixJITDylibDepInfo DepInfo;
525 for (auto &Dep : KV.second) {
526 auto HJ = HeaderAddrs.find(Dep);
527 if (HJ != HeaderAddrs.end())
528 DepInfo.push_back(HJ->second);
529 }
530 DIM.push_back(std::make_pair(H, std::move(DepInfo)));
531 }
532 SendResult(DIM);
533 return;
534 }
535
536 // Otherwise issue a lookup and re-run this phase when it completes.
537 lookupInitSymbolsAsync(
538 [this, SendResult = std::move(SendResult), JD](Error Err) mutable {
539 if (Err)
540 SendResult(std::move(Err));
541 else
542 pushInitializersLoop(std::move(SendResult), JD);
543 },
544 ES, std::move(NewInitSymbols));
545 }
546
rt_recordInitializers(PushInitializersSendResultFn SendResult,ExecutorAddr JDHeaderAddr)547 void ELFNixPlatform::rt_recordInitializers(
548 PushInitializersSendResultFn SendResult, ExecutorAddr JDHeaderAddr) {
549 JITDylibSP JD;
550 {
551 std::lock_guard<std::mutex> Lock(PlatformMutex);
552 auto I = HandleAddrToJITDylib.find(JDHeaderAddr);
553 if (I != HandleAddrToJITDylib.end())
554 JD = I->second;
555 }
556
557 LLVM_DEBUG({
558 dbgs() << "ELFNixPlatform::rt_recordInitializers(" << JDHeaderAddr << ") ";
559 if (JD)
560 dbgs() << "pushing initializers for " << JD->getName() << "\n";
561 else
562 dbgs() << "No JITDylib for header address.\n";
563 });
564
565 if (!JD) {
566 SendResult(make_error<StringError>("No JITDylib with header addr " +
567 formatv("{0:x}", JDHeaderAddr),
568 inconvertibleErrorCode()));
569 return;
570 }
571
572 pushInitializersLoop(std::move(SendResult), JD);
573 }
574
rt_lookupSymbol(SendSymbolAddressFn SendResult,ExecutorAddr Handle,StringRef SymbolName)575 void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
576 ExecutorAddr Handle,
577 StringRef SymbolName) {
578 LLVM_DEBUG({
579 dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n";
580 });
581
582 JITDylib *JD = nullptr;
583
584 {
585 std::lock_guard<std::mutex> Lock(PlatformMutex);
586 auto I = HandleAddrToJITDylib.find(Handle);
587 if (I != HandleAddrToJITDylib.end())
588 JD = I->second;
589 }
590
591 if (!JD) {
592 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
593 SendResult(make_error<StringError>("No JITDylib associated with handle " +
594 formatv("{0:x}", Handle),
595 inconvertibleErrorCode()));
596 return;
597 }
598
599 // Use functor class to work around XL build compiler issue on AIX.
600 class RtLookupNotifyComplete {
601 public:
602 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
603 : SendResult(std::move(SendResult)) {}
604 void operator()(Expected<SymbolMap> Result) {
605 if (Result) {
606 assert(Result->size() == 1 && "Unexpected result map count");
607 SendResult(Result->begin()->second.getAddress());
608 } else {
609 SendResult(Result.takeError());
610 }
611 }
612
613 private:
614 SendSymbolAddressFn SendResult;
615 };
616
617 ES.lookup(
618 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
619 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
620 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
621 }
622
bootstrapPipelineStart(jitlink::LinkGraph & G)623 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineStart(
624 jitlink::LinkGraph &G) {
625 // Increment the active graphs count in BootstrapInfo.
626 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
627 ++MP.Bootstrap.load()->ActiveGraphs;
628 return Error::success();
629 }
630
631 Error ELFNixPlatform::ELFNixPlatformPlugin::
bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph & G)632 bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) {
633 // Record bootstrap function names.
634 std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = {
635 {*MP.DSOHandleSymbol, &MP.Bootstrap.load()->ELFNixHeaderAddr},
636 {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr},
637 {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr},
638 {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr},
639 {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr},
640 {*MP.RegisterObjectSections.Name, &MP.RegisterObjectSections.Addr},
641 {*MP.DeregisterObjectSections.Name, &MP.DeregisterObjectSections.Addr},
642 {*MP.RegisterInitSections.Name, &MP.RegisterInitSections.Addr},
643 {*MP.DeregisterInitSections.Name, &MP.DeregisterInitSections.Addr},
644 {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}};
645
646 bool RegisterELFNixHeader = false;
647
648 for (auto *Sym : G.defined_symbols()) {
649 for (auto &RTSym : RuntimeSymbols) {
650 if (Sym->hasName() && *Sym->getName() == RTSym.first) {
651 if (*RTSym.second)
652 return make_error<StringError>(
653 "Duplicate " + RTSym.first +
654 " detected during ELFNixPlatform bootstrap",
655 inconvertibleErrorCode());
656
657 if (*Sym->getName() == *MP.DSOHandleSymbol)
658 RegisterELFNixHeader = true;
659
660 *RTSym.second = Sym->getAddress();
661 }
662 }
663 }
664
665 if (RegisterELFNixHeader) {
666 // If this graph defines the elfnix header symbol then create the internal
667 // mapping between it and PlatformJD.
668 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
669 MP.JITDylibToHandleAddr[&MP.PlatformJD] =
670 MP.Bootstrap.load()->ELFNixHeaderAddr;
671 MP.HandleAddrToJITDylib[MP.Bootstrap.load()->ELFNixHeaderAddr] =
672 &MP.PlatformJD;
673 }
674
675 return Error::success();
676 }
677
bootstrapPipelineEnd(jitlink::LinkGraph & G)678 Error ELFNixPlatform::ELFNixPlatformPlugin::bootstrapPipelineEnd(
679 jitlink::LinkGraph &G) {
680 std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex);
681 assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed");
682 --MP.Bootstrap.load()->ActiveGraphs;
683 // Notify Bootstrap->CV while holding the mutex because the mutex is
684 // also keeping Bootstrap->CV alive.
685 if (MP.Bootstrap.load()->ActiveGraphs == 0)
686 MP.Bootstrap.load()->CV.notify_all();
687 return Error::success();
688 }
689
registerPerObjectSections(jitlink::LinkGraph & G,const ELFPerObjectSectionsToRegister & POSR,bool IsBootstrapping)690 Error ELFNixPlatform::registerPerObjectSections(
691 jitlink::LinkGraph &G, const ELFPerObjectSectionsToRegister &POSR,
692 bool IsBootstrapping) {
693 using SPSRegisterPerObjSectionsArgs =
694 SPSArgList<SPSELFPerObjectSectionsToRegister>;
695
696 if (LLVM_UNLIKELY(IsBootstrapping)) {
697 Bootstrap.load()->addArgumentsToRTFnMap(
698 &RegisterObjectSections, &DeregisterObjectSections,
699 getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR),
700 getArgDataBufferType<SPSRegisterPerObjSectionsArgs>(POSR));
701 return Error::success();
702 }
703
704 G.allocActions().push_back(
705 {cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
706 RegisterObjectSections.Addr, POSR)),
707 cantFail(WrapperFunctionCall::Create<SPSRegisterPerObjSectionsArgs>(
708 DeregisterObjectSections.Addr, POSR))});
709
710 return Error::success();
711 }
712
createPThreadKey()713 Expected<uint64_t> ELFNixPlatform::createPThreadKey() {
714 if (!CreatePThreadKey.Addr)
715 return make_error<StringError>(
716 "Attempting to create pthread key in target, but runtime support has "
717 "not been loaded yet",
718 inconvertibleErrorCode());
719
720 Expected<uint64_t> Result(0);
721 if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>(
722 CreatePThreadKey.Addr, Result))
723 return std::move(Err);
724 return Result;
725 }
726
modifyPassConfig(MaterializationResponsibility & MR,jitlink::LinkGraph & LG,jitlink::PassConfiguration & Config)727 void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig(
728 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
729 jitlink::PassConfiguration &Config) {
730 using namespace jitlink;
731
732 bool InBootstrapPhase =
733 &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap;
734
735 // If we're in the bootstrap phase then increment the active graphs.
736 if (InBootstrapPhase) {
737 Config.PrePrunePasses.push_back(
738 [this](LinkGraph &G) { return bootstrapPipelineStart(G); });
739 Config.PostAllocationPasses.push_back([this](LinkGraph &G) {
740 return bootstrapPipelineRecordRuntimeFunctions(G);
741 });
742 }
743
744 // If the initializer symbol is the __dso_handle symbol then just add
745 // the DSO handle support passes.
746 if (auto InitSymbol = MR.getInitializerSymbol()) {
747 if (InitSymbol == MP.DSOHandleSymbol && !InBootstrapPhase) {
748 addDSOHandleSupportPasses(MR, Config);
749 // The DSOHandle materialization unit doesn't require any other
750 // support, so we can bail out early.
751 return;
752 }
753
754 /// Preserve init sections.
755 Config.PrePrunePasses.push_back(
756 [this, &MR](jitlink::LinkGraph &G) -> Error {
757 if (auto Err = preserveInitSections(G, MR))
758 return Err;
759 return Error::success();
760 });
761 }
762
763 // Add passes for eh-frame and TLV support.
764 addEHAndTLVSupportPasses(MR, Config, InBootstrapPhase);
765
766 // If the object contains initializers then add passes to record them.
767 Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib(),
768 InBootstrapPhase](jitlink::LinkGraph &G) {
769 return registerInitSections(G, JD, InBootstrapPhase);
770 });
771
772 // If we're in the bootstrap phase then steal allocation actions and then
773 // decrement the active graphs.
774 if (InBootstrapPhase)
775 Config.PostFixupPasses.push_back(
776 [this](LinkGraph &G) { return bootstrapPipelineEnd(G); });
777 }
778
addDSOHandleSupportPasses(MaterializationResponsibility & MR,jitlink::PassConfiguration & Config)779 void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses(
780 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) {
781
782 Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()](
783 jitlink::LinkGraph &G) -> Error {
784 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
785 return Sym->getName() == MP.DSOHandleSymbol;
786 });
787 assert(I != G.defined_symbols().end() && "Missing DSO handle symbol");
788 {
789 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
790 auto HandleAddr = (*I)->getAddress();
791 MP.HandleAddrToJITDylib[HandleAddr] = &JD;
792 MP.JITDylibToHandleAddr[&JD] = HandleAddr;
793
794 G.allocActions().push_back(
795 {cantFail(WrapperFunctionCall::Create<
796 SPSArgList<SPSString, SPSExecutorAddr>>(
797 MP.RegisterJITDylib.Addr, JD.getName(), HandleAddr)),
798 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
799 MP.DeregisterJITDylib.Addr, HandleAddr))});
800 }
801 return Error::success();
802 });
803 }
804
addEHAndTLVSupportPasses(MaterializationResponsibility & MR,jitlink::PassConfiguration & Config,bool IsBootstrapping)805 void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses(
806 MaterializationResponsibility &MR, jitlink::PassConfiguration &Config,
807 bool IsBootstrapping) {
808
809 // Insert TLV lowering at the start of the PostPrunePasses, since we want
810 // it to run before GOT/PLT lowering.
811
812 // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build
813 // pass has done. Because the TLS descriptor need to be allocate in GOT.
814 Config.PostPrunePasses.push_back(
815 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
816 return fixTLVSectionsAndEdges(G, JD);
817 });
818
819 // Add a pass to register the final addresses of the eh-frame and TLV sections
820 // with the runtime.
821 Config.PostFixupPasses.push_back([this, IsBootstrapping](
822 jitlink::LinkGraph &G) -> Error {
823 ELFPerObjectSectionsToRegister POSR;
824
825 if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) {
826 jitlink::SectionRange R(*EHFrameSection);
827 if (!R.empty())
828 POSR.EHFrameSection = R.getRange();
829 }
830
831 // Get a pointer to the thread data section if there is one. It will be used
832 // below.
833 jitlink::Section *ThreadDataSection =
834 G.findSectionByName(ELFThreadDataSectionName);
835
836 // Handle thread BSS section if there is one.
837 if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) {
838 // If there's already a thread data section in this graph then merge the
839 // thread BSS section content into it, otherwise just treat the thread
840 // BSS section as the thread data section.
841 if (ThreadDataSection)
842 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
843 else
844 ThreadDataSection = ThreadBSSSection;
845 }
846
847 // Having merged thread BSS (if present) and thread data (if present),
848 // record the resulting section range.
849 if (ThreadDataSection) {
850 jitlink::SectionRange R(*ThreadDataSection);
851 if (!R.empty())
852 POSR.ThreadDataSection = R.getRange();
853 }
854
855 if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) {
856 if (auto Err = MP.registerPerObjectSections(G, POSR, IsBootstrapping))
857 return Err;
858 }
859
860 return Error::success();
861 });
862 }
863
preserveInitSections(jitlink::LinkGraph & G,MaterializationResponsibility & MR)864 Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections(
865 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
866
867 if (const auto &InitSymName = MR.getInitializerSymbol()) {
868
869 jitlink::Symbol *InitSym = nullptr;
870
871 for (auto &InitSection : G.sections()) {
872 // Skip non-init sections.
873 if (!isELFInitializerSection(InitSection.getName()) ||
874 InitSection.empty())
875 continue;
876
877 // Create the init symbol if it has not been created already and attach it
878 // to the first block.
879 if (!InitSym) {
880 auto &B = **InitSection.blocks().begin();
881 InitSym = &G.addDefinedSymbol(
882 B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
883 jitlink::Scope::SideEffectsOnly, false, true);
884 }
885
886 // Add keep-alive edges to anonymous symbols in all other init blocks.
887 for (auto *B : InitSection.blocks()) {
888 if (B == &InitSym->getBlock())
889 continue;
890
891 auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
892 InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
893 }
894 }
895 }
896
897 return Error::success();
898 }
899
registerInitSections(jitlink::LinkGraph & G,JITDylib & JD,bool IsBootstrapping)900 Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections(
901 jitlink::LinkGraph &G, JITDylib &JD, bool IsBootstrapping) {
902 SmallVector<ExecutorAddrRange> ELFNixPlatformSecs;
903 LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n");
904
905 SmallVector<jitlink::Section *> OrderedInitSections;
906 for (auto &Sec : G.sections())
907 if (isELFInitializerSection(Sec.getName()))
908 OrderedInitSections.push_back(&Sec);
909
910 // FIXME: This handles priority order within the current graph, but we'll need
911 // to include priority information in the initializer allocation
912 // actions in order to respect the ordering across multiple graphs.
913 llvm::sort(OrderedInitSections, [](const jitlink::Section *LHS,
914 const jitlink::Section *RHS) {
915 if (LHS->getName().starts_with(".init_array")) {
916 if (RHS->getName().starts_with(".init_array")) {
917 StringRef LHSPrioStr(LHS->getName());
918 StringRef RHSPrioStr(RHS->getName());
919 uint64_t LHSPriority;
920 bool LHSHasPriority = LHSPrioStr.consume_front(".init_array.") &&
921 !LHSPrioStr.getAsInteger(10, LHSPriority);
922 uint64_t RHSPriority;
923 bool RHSHasPriority = RHSPrioStr.consume_front(".init_array.") &&
924 !RHSPrioStr.getAsInteger(10, RHSPriority);
925 if (LHSHasPriority)
926 return RHSHasPriority ? LHSPriority < RHSPriority : true;
927 else if (RHSHasPriority)
928 return false;
929 // If we get here we'll fall through to the
930 // LHS->getName() < RHS->getName() test below.
931 } else {
932 // .init_array[.N] comes before any non-.init_array[.N] section.
933 return true;
934 }
935 }
936 return LHS->getName() < RHS->getName();
937 });
938
939 for (auto &Sec : OrderedInitSections)
940 ELFNixPlatformSecs.push_back(jitlink::SectionRange(*Sec).getRange());
941
942 // Dump the scraped inits.
943 LLVM_DEBUG({
944 dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n";
945 for (auto &Sec : G.sections()) {
946 jitlink::SectionRange R(Sec);
947 dbgs() << " " << Sec.getName() << ": " << R.getRange() << "\n";
948 }
949 });
950
951 ExecutorAddr HeaderAddr;
952 {
953 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
954 auto I = MP.JITDylibToHandleAddr.find(&JD);
955 assert(I != MP.JITDylibToHandleAddr.end() && "No header registered for JD");
956 assert(I->second && "Null header registered for JD");
957 HeaderAddr = I->second;
958 }
959
960 using SPSRegisterInitSectionsArgs =
961 SPSArgList<SPSExecutorAddr, SPSSequence<SPSExecutorAddrRange>>;
962
963 if (LLVM_UNLIKELY(IsBootstrapping)) {
964 MP.Bootstrap.load()->addArgumentsToRTFnMap(
965 &MP.RegisterInitSections, &MP.DeregisterInitSections,
966 getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
967 ELFNixPlatformSecs),
968 getArgDataBufferType<SPSRegisterInitSectionsArgs>(HeaderAddr,
969 ELFNixPlatformSecs));
970 return Error::success();
971 }
972
973 G.allocActions().push_back(
974 {cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
975 MP.RegisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs)),
976 cantFail(WrapperFunctionCall::Create<SPSRegisterInitSectionsArgs>(
977 MP.DeregisterInitSections.Addr, HeaderAddr, ELFNixPlatformSecs))});
978
979 return Error::success();
980 }
981
fixTLVSectionsAndEdges(jitlink::LinkGraph & G,JITDylib & JD)982 Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges(
983 jitlink::LinkGraph &G, JITDylib &JD) {
984 auto TLSGetAddrSymbolName = G.intern("__tls_get_addr");
985 auto TLSDescResolveSymbolName = G.intern("__tlsdesc_resolver");
986 for (auto *Sym : G.external_symbols()) {
987 if (Sym->getName() == TLSGetAddrSymbolName) {
988 auto TLSGetAddr =
989 MP.getExecutionSession().intern("___orc_rt_elfnix_tls_get_addr");
990 Sym->setName(std::move(TLSGetAddr));
991 } else if (Sym->getName() == TLSDescResolveSymbolName) {
992 auto TLSGetAddr =
993 MP.getExecutionSession().intern("___orc_rt_elfnix_tlsdesc_resolver");
994 Sym->setName(std::move(TLSGetAddr));
995 }
996 }
997
998 auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO");
999
1000 if (TLSInfoEntrySection) {
1001 std::optional<uint64_t> Key;
1002 {
1003 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
1004 auto I = MP.JITDylibToPThreadKey.find(&JD);
1005 if (I != MP.JITDylibToPThreadKey.end())
1006 Key = I->second;
1007 }
1008 if (!Key) {
1009 if (auto KeyOrErr = MP.createPThreadKey())
1010 Key = *KeyOrErr;
1011 else
1012 return KeyOrErr.takeError();
1013 }
1014
1015 uint64_t PlatformKeyBits =
1016 support::endian::byte_swap(*Key, G.getEndianness());
1017
1018 for (auto *B : TLSInfoEntrySection->blocks()) {
1019 // FIXME: The TLS descriptor byte length may different with different
1020 // ISA
1021 assert(B->getSize() == (G.getPointerSize() * 2) &&
1022 "TLS descriptor must be 2 words length");
1023 auto TLSInfoEntryContent = B->getMutableContent(G);
1024 memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize());
1025 }
1026 }
1027
1028 return Error::success();
1029 }
1030
1031 } // End namespace orc.
1032 } // End namespace llvm.
1033