1 //===- elfnix_platform.cpp ------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the ELF-on-*IX runtime.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "elfnix_platform.h"
14 #include "common.h"
15 #include "compiler.h"
16 #include "error.h"
17 #include "jit_dispatch.h"
18 #include "record_section_tracker.h"
19 #include "wrapper_function_utils.h"
20
21 #include <algorithm>
22 #include <map>
23 #include <mutex>
24 #include <sstream>
25 #include <string_view>
26 #include <unordered_map>
27 #include <vector>
28
29 using namespace orc_rt;
30 using namespace orc_rt::elfnix;
31
32 // Declare function tags for functions in the JIT process.
33 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_reoptimize_tag)
34 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_push_initializers_tag)
35 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag)
36
37 // eh-frame registration functions, made available via aliases
38 // installed by the Platform
39 extern "C" void __register_frame(const void *);
40 extern "C" void __deregister_frame(const void *);
41
42 extern "C" void
43 __unw_add_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
44 extern "C" void
45 __unw_remove_dynamic_eh_frame_section(const void *) ORC_RT_WEAK_IMPORT;
46
47 namespace {
48
49 struct TLSInfoEntry {
50 unsigned long Key = 0;
51 unsigned long DataAddress = 0;
52 };
53
54 struct TLSDescriptor {
55 void (*Resolver)(void *);
56 TLSInfoEntry *InfoEntry;
57 };
58
59 class ELFNixPlatformRuntimeState {
60 private:
61 struct AtExitEntry {
62 void (*Func)(void *);
63 void *Arg;
64 };
65
66 using AtExitsVector = std::vector<AtExitEntry>;
67
68 struct PerJITDylibState {
69 std::string Name;
70 void *Header = nullptr;
71 size_t RefCount = 0;
72 size_t LinkedAgainstRefCount = 0;
73 bool AllowReinitialization = false;
74 AtExitsVector AtExits;
75 std::vector<PerJITDylibState *> Deps;
76 RecordSectionsTracker<void (*)()> RecordedInits;
77
referenced__anonb364b2400111::ELFNixPlatformRuntimeState::PerJITDylibState78 bool referenced() const {
79 return LinkedAgainstRefCount != 0 || RefCount != 0;
80 }
81 };
82
83 public:
84 static void initialize(void *DSOHandle);
85 static ELFNixPlatformRuntimeState &get();
86 static void destroy();
87
88 ELFNixPlatformRuntimeState(void *DSOHandle);
89
90 // Delete copy and move constructors.
91 ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete;
92 ELFNixPlatformRuntimeState &
93 operator=(const ELFNixPlatformRuntimeState &) = delete;
94 ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete;
95 ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete;
96
97 Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR);
98 Error registerJITDylib(std::string &Name, void *Handle);
99 Error deregisterJITDylib(void *Handle);
100 Error registerInits(ExecutorAddr HeaderAddr,
101 std::vector<ExecutorAddrRange> Inits);
102 Error deregisterInits(ExecutorAddr HeaderAddr,
103 std::vector<ExecutorAddrRange> Inits);
104 Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR);
105
106 const char *dlerror();
107 void *dlopen(std::string_view Name, int Mode);
108 int dlupdate(void *DSOHandle);
109 int dlclose(void *DSOHandle);
110 void *dlsym(void *DSOHandle, std::string_view Symbol);
111
112 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
113 void runAtExits(void *DSOHandle);
114 void runAtExits(std::unique_lock<std::recursive_mutex> &JDStateLock,
115 PerJITDylibState &JDS);
116
117 /// Returns the base address of the section containing ThreadData.
118 Expected<std::pair<const char *, size_t>>
119 getThreadDataSectionFor(const char *ThreadData);
120
getPlatformJDDSOHandle()121 void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; }
122
123 private:
124 PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle);
125 PerJITDylibState *getJITDylibStateByName(std::string_view Path);
126
127 Error registerThreadDataSection(span<const char> ThreadDataSection);
128
129 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle,
130 std::string_view Symbol);
131
132 Error runInits(std::unique_lock<std::recursive_mutex> &JDStatesLock,
133 PerJITDylibState &JDS);
134 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
135 Error dlopenFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
136 PerJITDylibState &JDS);
137 Error dlopenInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
138 PerJITDylibState &JDS,
139 ELFNixJITDylibDepInfoMap &DepInfo);
140 Error dlupdateImpl(void *DSOHandle);
141 Error dlupdateFull(std::unique_lock<std::recursive_mutex> &JDStatesLock,
142 PerJITDylibState &JDS);
143
144 Error dlcloseImpl(void *DSOHandle);
145 Error dlcloseInitialize(std::unique_lock<std::recursive_mutex> &JDStatesLock,
146 PerJITDylibState &JDS);
147
148 static ELFNixPlatformRuntimeState *MOPS;
149
150 void *PlatformJDDSOHandle;
151
152 // Frame registration functions:
153 void (*registerEHFrameSection)(const void *) = nullptr;
154 void (*deregisterEHFrameSection)(const void *) = nullptr;
155
156 // FIXME: Move to thread-state.
157 std::string DLFcnError;
158
159 std::recursive_mutex JDStatesMutex;
160 std::unordered_map<void *, PerJITDylibState> JDStates;
161 std::unordered_map<std::string, void *> JDNameToHeader;
162
163 std::mutex ThreadDataSectionsMutex;
164 std::map<const char *, size_t> ThreadDataSections;
165 };
166
167 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr;
168
initialize(void * DSOHandle)169 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) {
170 assert(!MOPS && "ELFNixPlatformRuntimeState should be null");
171 MOPS = new ELFNixPlatformRuntimeState(DSOHandle);
172 }
173
get()174 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() {
175 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
176 return *MOPS;
177 }
178
destroy()179 void ELFNixPlatformRuntimeState::destroy() {
180 assert(MOPS && "ELFNixPlatformRuntimeState not initialized");
181 delete MOPS;
182 }
183
ELFNixPlatformRuntimeState(void * DSOHandle)184 ELFNixPlatformRuntimeState::ELFNixPlatformRuntimeState(void *DSOHandle)
185 : PlatformJDDSOHandle(DSOHandle) {
186 if (__unw_add_dynamic_eh_frame_section &&
187 __unw_remove_dynamic_eh_frame_section) {
188 registerEHFrameSection = __unw_add_dynamic_eh_frame_section;
189 deregisterEHFrameSection = __unw_remove_dynamic_eh_frame_section;
190 } else {
191 registerEHFrameSection = __register_frame;
192 deregisterEHFrameSection = __deregister_frame;
193 }
194 }
195
registerObjectSections(ELFNixPerObjectSectionsToRegister POSR)196 Error ELFNixPlatformRuntimeState::registerObjectSections(
197 ELFNixPerObjectSectionsToRegister POSR) {
198 if (POSR.EHFrameSection.Start)
199 registerEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
200
201 if (POSR.ThreadDataSection.Start) {
202 if (auto Err = registerThreadDataSection(
203 POSR.ThreadDataSection.toSpan<const char>()))
204 return Err;
205 }
206
207 return Error::success();
208 }
209
deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR)210 Error ELFNixPlatformRuntimeState::deregisterObjectSections(
211 ELFNixPerObjectSectionsToRegister POSR) {
212 if (POSR.EHFrameSection.Start)
213 deregisterEHFrameSection(POSR.EHFrameSection.Start.toPtr<const char *>());
214
215 return Error::success();
216 }
217
registerJITDylib(std::string & Name,void * Handle)218 Error ELFNixPlatformRuntimeState::registerJITDylib(std::string &Name,
219 void *Handle) {
220 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
221
222 if (JDStates.count(Handle)) {
223 std::ostringstream ErrStream;
224 ErrStream << "Duplicate JITDylib registration for header " << Handle
225 << " (name = " << Name << ")";
226 return make_error<StringError>(ErrStream.str());
227 }
228
229 if (JDNameToHeader.count(Name)) {
230 std::ostringstream ErrStream;
231 ErrStream << "Duplicate JITDylib registration for header " << Handle
232 << " (header = " << Handle << ")";
233 return make_error<StringError>(ErrStream.str());
234 }
235
236 auto &JD = JDStates[Handle];
237 JD.Header = Handle;
238 JD.Name = std::move(Name);
239 JDNameToHeader[JD.Name] = Handle;
240 return Error::success();
241 }
242
deregisterJITDylib(void * Handle)243 Error ELFNixPlatformRuntimeState::deregisterJITDylib(void *Handle) {
244 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
245
246 auto I = JDStates.find(Handle);
247 if (I == JDStates.end()) {
248 std::ostringstream ErrStream;
249 ErrStream << "Attempted to deregister unrecognized header " << Handle;
250 return make_error<StringError>(ErrStream.str());
251 }
252
253 auto J = JDNameToHeader.find(
254 std::string(I->second.Name.data(), I->second.Name.size()));
255 assert(J != JDNameToHeader.end() &&
256 "Missing JDNameToHeader entry for JITDylib");
257 JDNameToHeader.erase(J);
258 JDStates.erase(I);
259 return Error::success();
260 }
261
registerInits(ExecutorAddr HeaderAddr,std::vector<ExecutorAddrRange> Inits)262 Error ELFNixPlatformRuntimeState::registerInits(
263 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
264 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
265 PerJITDylibState *JDS =
266 getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
267
268 if (!JDS) {
269 std::ostringstream ErrStream;
270 ErrStream << "Could not register object platform sections for "
271 "unrecognized header "
272 << HeaderAddr.toPtr<void *>();
273 return make_error<StringError>(ErrStream.str());
274 }
275
276 for (auto &I : Inits) {
277 JDS->RecordedInits.add(I.toSpan<void (*)()>());
278 }
279
280 return Error::success();
281 }
282
deregisterInits(ExecutorAddr HeaderAddr,std::vector<ExecutorAddrRange> Inits)283 Error ELFNixPlatformRuntimeState::deregisterInits(
284 ExecutorAddr HeaderAddr, std::vector<ExecutorAddrRange> Inits) {
285
286 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
287 PerJITDylibState *JDS =
288 getJITDylibStateByHeaderAddr(HeaderAddr.toPtr<void *>());
289
290 if (!JDS) {
291 std::ostringstream ErrStream;
292 ErrStream << "Could not register object platform sections for unrecognized "
293 "header "
294 << HeaderAddr.toPtr<void *>();
295 return make_error<StringError>(ErrStream.str());
296 }
297
298 for (auto &I : Inits) {
299 JDS->RecordedInits.removeIfPresent(I);
300 }
301
302 return Error::success();
303 }
304
dlerror()305 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
306
dlopen(std::string_view Path,int Mode)307 void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
308 if (auto H = dlopenImpl(Path, Mode))
309 return *H;
310 else {
311 // FIXME: Make dlerror thread safe.
312 DLFcnError = toString(H.takeError());
313 return nullptr;
314 }
315 }
316
dlupdate(void * DSOHandle)317 int ELFNixPlatformRuntimeState::dlupdate(void *DSOHandle) {
318 if (auto Err = dlupdateImpl(DSOHandle)) {
319 // FIXME: Make dlerror thread safe.
320 DLFcnError = toString(std::move(Err));
321 return -1;
322 }
323 return 0;
324 }
325
dlclose(void * DSOHandle)326 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) {
327 if (auto Err = dlcloseImpl(DSOHandle)) {
328 DLFcnError = toString(std::move(Err));
329 return -1;
330 }
331 return 0;
332 }
333
dlsym(void * DSOHandle,std::string_view Symbol)334 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle,
335 std::string_view Symbol) {
336 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol);
337 if (!Addr) {
338 DLFcnError = toString(Addr.takeError());
339 return 0;
340 }
341
342 return Addr->toPtr<void *>();
343 }
344
registerAtExit(void (* F)(void *),void * Arg,void * DSOHandle)345 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
346 void *DSOHandle) {
347 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
348 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex);
349 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
350 assert(JDS && "JITDylib state not initialized");
351 JDS->AtExits.push_back({F, Arg});
352 return 0;
353 }
354
runAtExits(void * DSOHandle)355 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) {
356 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
357 PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
358
359 if (JDS)
360 runAtExits(Lock, *JDS);
361 }
362
runAtExits(std::unique_lock<std::recursive_mutex> & JDStateLock,PerJITDylibState & JDS)363 void ELFNixPlatformRuntimeState::runAtExits(
364 std::unique_lock<std::recursive_mutex> &JDStateLock,
365 PerJITDylibState &JDS) {
366 AtExitsVector V = std::move(JDS.AtExits);
367
368 while (!V.empty()) {
369 auto &AE = V.back();
370 AE.Func(AE.Arg);
371 V.pop_back();
372 }
373 }
374
375 Expected<std::pair<const char *, size_t>>
getThreadDataSectionFor(const char * ThreadData)376 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
377 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
378 auto I = ThreadDataSections.upper_bound(ThreadData);
379 // Check that we have a valid entry conovering this address.
380 if (I == ThreadDataSections.begin())
381 return make_error<StringError>("No thread local data section for key");
382 I = std::prev(I);
383 if (ThreadData >= I->first + I->second)
384 return make_error<StringError>("No thread local data section for key");
385 return *I;
386 }
387
388 ELFNixPlatformRuntimeState::PerJITDylibState *
getJITDylibStateByHeaderAddr(void * DSOHandle)389 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) {
390 auto I = JDStates.find(DSOHandle);
391 if (I == JDStates.end())
392 return nullptr;
393
394 return &I->second;
395 }
396
397 ELFNixPlatformRuntimeState::PerJITDylibState *
getJITDylibStateByName(std::string_view Name)398 ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
399 // FIXME: Avoid creating string copy here.
400 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
401 if (I == JDNameToHeader.end())
402 return nullptr;
403 void *H = I->second;
404 auto J = JDStates.find(H);
405 assert(J != JDStates.end() &&
406 "JITDylib has name map entry but no header map entry");
407 return &J->second;
408 }
409
registerThreadDataSection(span<const char> ThreadDataSection)410 Error ELFNixPlatformRuntimeState::registerThreadDataSection(
411 span<const char> ThreadDataSection) {
412 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
413 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
414 if (I != ThreadDataSections.begin()) {
415 auto J = std::prev(I);
416 if (J->first + J->second > ThreadDataSection.data())
417 return make_error<StringError>("Overlapping .tdata sections");
418 }
419 ThreadDataSections.insert(
420 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
421 return Error::success();
422 }
423
424 Expected<ExecutorAddr>
lookupSymbolInJITDylib(void * DSOHandle,std::string_view Sym)425 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle,
426 std::string_view Sym) {
427 Expected<ExecutorAddr> Result((ExecutorAddr()));
428 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>(
429 SPSExecutorAddr,
430 SPSString)>::call(JITDispatch(&__orc_rt_elfnix_symbol_lookup_tag),
431 Result, ExecutorAddr::fromPtr(DSOHandle), Sym))
432 return std::move(Err);
433 return Result;
434 }
435
runInits(std::unique_lock<std::recursive_mutex> & JDStatesLock,PerJITDylibState & JDS)436 Error ELFNixPlatformRuntimeState::runInits(
437 std::unique_lock<std::recursive_mutex> &JDStatesLock,
438 PerJITDylibState &JDS) {
439 std::vector<span<void (*)()>> InitSections;
440 InitSections.reserve(JDS.RecordedInits.numNewSections());
441
442 JDS.RecordedInits.processNewSections(
443 [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
444
445 JDStatesLock.unlock();
446 for (auto Sec : InitSections)
447 for (auto *Init : Sec)
448 Init();
449
450 JDStatesLock.lock();
451
452 return Error::success();
453 }
454
dlopenImpl(std::string_view Path,int Mode)455 Expected<void *> ELFNixPlatformRuntimeState::dlopenImpl(std::string_view Path,
456 int Mode) {
457 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
458 PerJITDylibState *JDS = getJITDylibStateByName(Path);
459
460 if (!JDS)
461 return make_error<StringError>("No registered JTIDylib for path " +
462 std::string(Path.data(), Path.size()));
463
464 if (auto Err = dlopenFull(Lock, *JDS))
465 return std::move(Err);
466
467 ++JDS->RefCount;
468
469 return JDS->Header;
470 }
471
dlopenFull(std::unique_lock<std::recursive_mutex> & JDStateLock,PerJITDylibState & JDS)472 Error ELFNixPlatformRuntimeState::dlopenFull(
473 std::unique_lock<std::recursive_mutex> &JDStateLock,
474 PerJITDylibState &JDS) {
475 Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
476 JDStateLock.unlock();
477 if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
478 SPSExecutorAddr)>::
479 call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
480 ExecutorAddr::fromPtr(JDS.Header)))
481 return Err;
482 JDStateLock.lock();
483
484 if (!DepInfo)
485 return DepInfo.takeError();
486
487 if (auto Err = dlopenInitialize(JDStateLock, JDS, *DepInfo))
488 return Err;
489
490 if (!DepInfo->empty()) {
491 std::ostringstream ErrStream;
492 ErrStream << "Encountered unrecognized dep-info key headers "
493 "while processing dlopen of "
494 << JDS.Name;
495 return make_error<StringError>(ErrStream.str());
496 }
497
498 return Error::success();
499 }
500
dlopenInitialize(std::unique_lock<std::recursive_mutex> & JDStatesLock,PerJITDylibState & JDS,ELFNixJITDylibDepInfoMap & DepInfo)501 Error ELFNixPlatformRuntimeState::dlopenInitialize(
502 std::unique_lock<std::recursive_mutex> &JDStatesLock, PerJITDylibState &JDS,
503 ELFNixJITDylibDepInfoMap &DepInfo) {
504
505 auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
506 if (I == DepInfo.end())
507 return Error::success();
508
509 auto Deps = std::move(I->second);
510 DepInfo.erase(I);
511
512 std::vector<PerJITDylibState *> OldDeps;
513 std::swap(JDS.Deps, OldDeps);
514 JDS.Deps.reserve(Deps.size());
515 for (auto H : Deps) {
516 PerJITDylibState *DepJDS = getJITDylibStateByHeaderAddr(H.toPtr<void *>());
517 if (!DepJDS) {
518 std::ostringstream ErrStream;
519 ErrStream << "Encountered unrecognized dep header " << H.toPtr<void *>()
520 << " while initializing " << JDS.Name;
521 return make_error<StringError>(ErrStream.str());
522 }
523 ++DepJDS->LinkedAgainstRefCount;
524 if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
525 return Err;
526 }
527
528 if (auto Err = runInits(JDStatesLock, JDS))
529 return Err;
530
531 for (auto *DepJDS : OldDeps) {
532 --DepJDS->LinkedAgainstRefCount;
533 if (!DepJDS->referenced())
534 if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
535 return Err;
536 }
537 return Error::success();
538 }
539
dlupdateImpl(void * DSOHandle)540 Error ELFNixPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
541 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
542
543 // Try to find JITDylib state by name.
544 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
545
546 if (!JDS) {
547 std::ostringstream ErrStream;
548 ErrStream << "No registered JITDylib for " << DSOHandle;
549 return make_error<StringError>(ErrStream.str());
550 }
551
552 if (!JDS->referenced())
553 return make_error<StringError>("dlupdate failed, JITDylib must be open.");
554
555 if (auto Err = dlupdateFull(Lock, *JDS))
556 return Err;
557
558 return Error::success();
559 }
560
dlupdateFull(std::unique_lock<std::recursive_mutex> & JDStatesLock,PerJITDylibState & JDS)561 Error ELFNixPlatformRuntimeState::dlupdateFull(
562 std::unique_lock<std::recursive_mutex> &JDStatesLock,
563 PerJITDylibState &JDS) {
564 // Call back to the JIT to push the initializers.
565 Expected<ELFNixJITDylibDepInfoMap> DepInfo((ELFNixJITDylibDepInfoMap()));
566 // Unlock so that we can accept the initializer update.
567 JDStatesLock.unlock();
568 if (auto Err = WrapperFunction<SPSExpected<SPSELFNixJITDylibDepInfoMap>(
569 SPSExecutorAddr)>::
570 call(JITDispatch(&__orc_rt_elfnix_push_initializers_tag), DepInfo,
571 ExecutorAddr::fromPtr(JDS.Header)))
572 return Err;
573 JDStatesLock.lock();
574
575 if (!DepInfo)
576 return DepInfo.takeError();
577
578 if (auto Err = runInits(JDStatesLock, JDS))
579 return Err;
580
581 return Error::success();
582 }
583
dlcloseImpl(void * DSOHandle)584 Error ELFNixPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
585
586 std::unique_lock<std::recursive_mutex> Lock(JDStatesMutex);
587 PerJITDylibState *JDS = getJITDylibStateByHeaderAddr(DSOHandle);
588
589 if (!JDS) {
590 std::ostringstream ErrStream;
591 ErrStream << "No registered JITDylib for " << DSOHandle;
592 return make_error<StringError>(ErrStream.str());
593 }
594
595 --JDS->RefCount;
596
597 if (!JDS->referenced())
598 return dlcloseInitialize(Lock, *JDS);
599
600 return Error::success();
601 }
602
dlcloseInitialize(std::unique_lock<std::recursive_mutex> & JDStatesLock,PerJITDylibState & JDS)603 Error ELFNixPlatformRuntimeState::dlcloseInitialize(
604 std::unique_lock<std::recursive_mutex> &JDStatesLock,
605 PerJITDylibState &JDS) {
606 runAtExits(JDStatesLock, JDS);
607 JDS.RecordedInits.reset();
608 for (auto *DepJDS : JDS.Deps)
609 if (!JDS.referenced())
610 if (auto Err = dlcloseInitialize(JDStatesLock, *DepJDS))
611 return Err;
612
613 return Error::success();
614 }
615
616 class ELFNixPlatformRuntimeTLVManager {
617 public:
618 void *getInstance(const char *ThreadData);
619
620 private:
621 std::unordered_map<const char *, char *> Instances;
622 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
623 };
624
getInstance(const char * ThreadData)625 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
626 auto I = Instances.find(ThreadData);
627 if (I != Instances.end())
628 return I->second;
629 auto TDS =
630 ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
631 if (!TDS) {
632 __orc_rt_log_error(toString(TDS.takeError()).c_str());
633 return nullptr;
634 }
635
636 auto &Allocated = AllocatedSections[TDS->first];
637 if (!Allocated) {
638 Allocated = std::make_unique<char[]>(TDS->second);
639 memcpy(Allocated.get(), TDS->first, TDS->second);
640 }
641 size_t ThreadDataDelta = ThreadData - TDS->first;
642 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
643
644 char *Instance = Allocated.get() + ThreadDataDelta;
645 Instances[ThreadData] = Instance;
646 return Instance;
647 }
648
destroyELFNixTLVMgr(void * ELFNixTLVMgr)649 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) {
650 delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr);
651 }
652
653 } // end anonymous namespace
654
655 //------------------------------------------------------------------------------
656 // JIT entry points
657 //------------------------------------------------------------------------------
658
659 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_platform_bootstrap(char * ArgData,size_t ArgSize)660 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) {
661 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
662 ArgData, ArgSize,
663 [](ExecutorAddr DSOHandle) {
664 ELFNixPlatformRuntimeState::initialize(
665 DSOHandle.toPtr<void *>());
666 return Error::success();
667 })
668 .release();
669 }
670
671 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_platform_shutdown(char * ArgData,size_t ArgSize)672 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) {
673 return WrapperFunction<SPSError()>::handle(
674 ArgData, ArgSize,
675 []() {
676 ELFNixPlatformRuntimeState::destroy();
677 return Error::success();
678 })
679 .release();
680 }
681
682 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_register_jitdylib(char * ArgData,size_t ArgSize)683 __orc_rt_elfnix_register_jitdylib(char *ArgData, size_t ArgSize) {
684 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
685 ArgData, ArgSize,
686 [](std::string &JDName, ExecutorAddr HeaderAddr) {
687 return ELFNixPlatformRuntimeState::get().registerJITDylib(
688 JDName, HeaderAddr.toPtr<void *>());
689 })
690 .release();
691 }
692
693 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_deregister_jitdylib(char * ArgData,size_t ArgSize)694 __orc_rt_elfnix_deregister_jitdylib(char *ArgData, size_t ArgSize) {
695 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
696 ArgData, ArgSize,
697 [](ExecutorAddr HeaderAddr) {
698 return ELFNixPlatformRuntimeState::get().deregisterJITDylib(
699 HeaderAddr.toPtr<void *>());
700 })
701 .release();
702 }
703
704 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_register_init_sections(char * ArgData,size_t ArgSize)705 __orc_rt_elfnix_register_init_sections(char *ArgData, size_t ArgSize) {
706 return WrapperFunction<SPSError(SPSExecutorAddr,
707 SPSSequence<SPSExecutorAddrRange>)>::
708 handle(ArgData, ArgSize,
709 [](ExecutorAddr HeaderAddr,
710 std::vector<ExecutorAddrRange> &Inits) {
711 return ELFNixPlatformRuntimeState::get().registerInits(
712 HeaderAddr, std::move(Inits));
713 })
714 .release();
715 }
716
717 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_deregister_init_sections(char * ArgData,size_t ArgSize)718 __orc_rt_elfnix_deregister_init_sections(char *ArgData, size_t ArgSize) {
719 return WrapperFunction<SPSError(SPSExecutorAddr,
720 SPSSequence<SPSExecutorAddrRange>)>::
721 handle(ArgData, ArgSize,
722 [](ExecutorAddr HeaderAddr,
723 std::vector<ExecutorAddrRange> &Inits) {
724 return ELFNixPlatformRuntimeState::get().deregisterInits(
725 HeaderAddr, std::move(Inits));
726 })
727 .release();
728 }
729
730 /// Wrapper function for registering metadata on a per-object basis.
731 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_register_object_sections(char * ArgData,size_t ArgSize)732 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) {
733 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
734 handle(ArgData, ArgSize,
735 [](ELFNixPerObjectSectionsToRegister &POSR) {
736 return ELFNixPlatformRuntimeState::get().registerObjectSections(
737 std::move(POSR));
738 })
739 .release();
740 }
741
742 /// Wrapper for releasing per-object metadat.
743 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_deregister_object_sections(char * ArgData,size_t ArgSize)744 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) {
745 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>::
746 handle(ArgData, ArgSize,
747 [](ELFNixPerObjectSectionsToRegister &POSR) {
748 return ELFNixPlatformRuntimeState::get()
749 .deregisterObjectSections(std::move(POSR));
750 })
751 .release();
752 }
753
754 //------------------------------------------------------------------------------
755 // TLV support
756 //------------------------------------------------------------------------------
757
__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry * D)758 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) {
759 auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>(
760 pthread_getspecific(D->Key));
761 if (!TLVMgr)
762 TLVMgr = new ELFNixPlatformRuntimeTLVManager();
763 if (pthread_setspecific(D->Key, TLVMgr)) {
764 __orc_rt_log_error("Call to pthread_setspecific failed");
765 return nullptr;
766 }
767
768 return TLVMgr->getInstance(
769 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
770 }
771
___orc_rt_elfnix_tlsdesc_resolver_impl(TLSDescriptor * D,const char * ThreadPointer)772 ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl(
773 TLSDescriptor *D, const char *ThreadPointer) {
774 const char *TLVPtr = reinterpret_cast<const char *>(
775 __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry));
776 return TLVPtr - ThreadPointer;
777 }
778
779 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_elfnix_create_pthread_key(char * ArgData,size_t ArgSize)780 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) {
781 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
782 ArgData, ArgSize,
783 []() -> Expected<uint64_t> {
784 pthread_key_t Key;
785 if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) {
786 __orc_rt_log_error("Call to pthread_key_create failed");
787 return make_error<StringError>(strerror(Err));
788 }
789 return static_cast<uint64_t>(Key);
790 })
791 .release();
792 }
793
794 //------------------------------------------------------------------------------
795 // cxa_atexit support
796 //------------------------------------------------------------------------------
797
__orc_rt_elfnix_cxa_atexit(void (* func)(void *),void * arg,void * dso_handle)798 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg,
799 void *dso_handle) {
800 return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg,
801 dso_handle);
802 }
803
__orc_rt_elfnix_atexit(void (* func)(void *))804 int __orc_rt_elfnix_atexit(void (*func)(void *)) {
805 auto &PlatformRTState = ELFNixPlatformRuntimeState::get();
806 return ELFNixPlatformRuntimeState::get().registerAtExit(
807 func, NULL, PlatformRTState.getPlatformJDDSOHandle());
808 }
809
__orc_rt_elfnix_cxa_finalize(void * dso_handle)810 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) {
811 ELFNixPlatformRuntimeState::get().runAtExits(dso_handle);
812 }
813
814 //------------------------------------------------------------------------------
815 // JIT'd dlfcn alternatives.
816 //------------------------------------------------------------------------------
817
__orc_rt_elfnix_jit_dlerror()818 const char *__orc_rt_elfnix_jit_dlerror() {
819 return ELFNixPlatformRuntimeState::get().dlerror();
820 }
821
__orc_rt_elfnix_jit_dlopen(const char * path,int mode)822 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) {
823 return ELFNixPlatformRuntimeState::get().dlopen(path, mode);
824 }
825
__orc_rt_elfnix_jit_dlupdate(void * dso_handle)826 int __orc_rt_elfnix_jit_dlupdate(void *dso_handle) {
827 return ELFNixPlatformRuntimeState::get().dlupdate(dso_handle);
828 }
829
__orc_rt_elfnix_jit_dlclose(void * dso_handle)830 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) {
831 return ELFNixPlatformRuntimeState::get().dlclose(dso_handle);
832 }
833
__orc_rt_elfnix_jit_dlsym(void * dso_handle,const char * symbol)834 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) {
835 return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol);
836 }
837
838 //------------------------------------------------------------------------------
839 // ELFNix Run Program
840 //------------------------------------------------------------------------------
841
__orc_rt_elfnix_run_program(const char * JITDylibName,const char * EntrySymbolName,int argc,char * argv[])842 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program(
843 const char *JITDylibName, const char *EntrySymbolName, int argc,
844 char *argv[]) {
845 using MainTy = int (*)(int, char *[]);
846
847 void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName,
848 orc_rt::elfnix::ORC_RT_RTLD_LAZY);
849 if (!H) {
850 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
851 return -1;
852 }
853
854 auto *Main =
855 reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName));
856
857 if (!Main) {
858 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
859 return -1;
860 }
861
862 int Result = Main(argc, argv);
863
864 if (__orc_rt_elfnix_jit_dlclose(H) == -1)
865 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror());
866
867 return Result;
868 }
869