1 //===- elfnix_platform.cpp ------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains code required to load the rest of the ELF-on-*IX runtime. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "elfnix_platform.h" 14 #include "common.h" 15 #include "error.h" 16 #include "wrapper_function_utils.h" 17 18 #include <algorithm> 19 #include <map> 20 #include <mutex> 21 #include <sstream> 22 #include <unordered_map> 23 #include <vector> 24 25 using namespace __orc_rt; 26 using namespace __orc_rt::elfnix; 27 28 // Declare function tags for functions in the JIT process. 29 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag) 30 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag) 31 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag) 32 33 // eh-frame registration functions, made available via aliases 34 // installed by the Platform 35 extern "C" void __orc_rt_register_eh_frame_section(const void *); 36 extern "C" void __orc_rt_deregister_eh_frame_section(const void *); 37 38 namespace { 39 40 Error validatePointerSectionExtent(const char *SectionName, 41 const ExecutorAddrRange &SE) { 42 if (SE.size().getValue() % sizeof(uintptr_t)) { 43 std::ostringstream ErrMsg; 44 ErrMsg << std::hex << "Size of " << SectionName << " 0x" 45 << SE.Start.getValue() << " -- 0x" << SE.End.getValue() 46 << " is not a pointer multiple"; 47 return make_error<StringError>(ErrMsg.str()); 48 } 49 return Error::success(); 50 } 51 52 Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections, 53 const ELFNixJITDylibInitializers &MOJDIs) { 54 55 for (const auto &ModInits : InitArraySections) { 56 if (auto Err = validatePointerSectionExtent(".init_array", ModInits)) 57 return Err; 58 59 using InitFunc = void (*)(); 60 for (auto *Init : ModInits.toSpan<InitFunc>()) 61 (*Init)(); 62 } 63 64 return Error::success(); 65 } 66 67 struct TLSInfoEntry { 68 unsigned long Key = 0; 69 unsigned long DataAddress = 0; 70 }; 71 72 struct TLSDescriptor { 73 void (*Resolver)(void *); 74 TLSInfoEntry *InfoEntry; 75 }; 76 77 class ELFNixPlatformRuntimeState { 78 private: 79 struct AtExitEntry { 80 void (*Func)(void *); 81 void *Arg; 82 }; 83 84 using AtExitsVector = std::vector<AtExitEntry>; 85 86 struct PerJITDylibState { 87 void *Header = nullptr; 88 size_t RefCount = 0; 89 bool AllowReinitialization = false; 90 AtExitsVector AtExits; 91 }; 92 93 public: 94 static void initialize(void *DSOHandle); 95 static ELFNixPlatformRuntimeState &get(); 96 static void destroy(); 97 98 ELFNixPlatformRuntimeState(void *DSOHandle) 99 : PlatformJDDSOHandle(DSOHandle) {} 100 101 // Delete copy and move constructors. 102 ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete; 103 ELFNixPlatformRuntimeState & 104 operator=(const ELFNixPlatformRuntimeState &) = delete; 105 ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete; 106 ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete; 107 108 Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR); 109 Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR); 110 111 const char *dlerror(); 112 void *dlopen(string_view Name, int Mode); 113 int dlclose(void *DSOHandle); 114 void *dlsym(void *DSOHandle, string_view Symbol); 115 116 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); 117 void runAtExits(void *DSOHandle); 118 119 /// Returns the base address of the section containing ThreadData. 120 Expected<std::pair<const char *, size_t>> 121 getThreadDataSectionFor(const char *ThreadData); 122 123 void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; } 124 125 private: 126 PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); 127 PerJITDylibState *getJITDylibStateByName(string_view Path); 128 PerJITDylibState & 129 getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs); 130 131 Error registerThreadDataSection(span<const char> ThreadDataSection); 132 133 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, 134 string_view Symbol); 135 136 Expected<ELFNixJITDylibInitializerSequence> 137 getJITDylibInitializersByName(string_view Path); 138 Expected<void *> dlopenInitialize(string_view Path, int Mode); 139 Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs); 140 141 static ELFNixPlatformRuntimeState *MOPS; 142 143 void *PlatformJDDSOHandle; 144 145 // FIXME: Move to thread-state. 146 std::string DLFcnError; 147 148 std::recursive_mutex JDStatesMutex; 149 std::unordered_map<void *, PerJITDylibState> JDStates; 150 std::unordered_map<std::string, void *> JDNameToHeader; 151 152 std::mutex ThreadDataSectionsMutex; 153 std::map<const char *, size_t> ThreadDataSections; 154 }; 155 156 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr; 157 158 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) { 159 assert(!MOPS && "ELFNixPlatformRuntimeState should be null"); 160 MOPS = new ELFNixPlatformRuntimeState(DSOHandle); 161 } 162 163 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() { 164 assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); 165 return *MOPS; 166 } 167 168 void ELFNixPlatformRuntimeState::destroy() { 169 assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); 170 delete MOPS; 171 } 172 173 Error ELFNixPlatformRuntimeState::registerObjectSections( 174 ELFNixPerObjectSectionsToRegister POSR) { 175 if (POSR.EHFrameSection.Start) 176 __orc_rt_register_eh_frame_section( 177 POSR.EHFrameSection.Start.toPtr<const char *>()); 178 179 if (POSR.ThreadDataSection.Start) { 180 if (auto Err = registerThreadDataSection( 181 POSR.ThreadDataSection.toSpan<const char>())) 182 return Err; 183 } 184 185 return Error::success(); 186 } 187 188 Error ELFNixPlatformRuntimeState::deregisterObjectSections( 189 ELFNixPerObjectSectionsToRegister POSR) { 190 if (POSR.EHFrameSection.Start) 191 __orc_rt_deregister_eh_frame_section( 192 POSR.EHFrameSection.Start.toPtr<const char *>()); 193 194 return Error::success(); 195 } 196 197 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } 198 199 void *ELFNixPlatformRuntimeState::dlopen(string_view Path, int Mode) { 200 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 201 202 // Use fast path if all JITDylibs are already loaded and don't require 203 // re-running initializers. 204 if (auto *JDS = getJITDylibStateByName(Path)) { 205 if (!JDS->AllowReinitialization) { 206 ++JDS->RefCount; 207 return JDS->Header; 208 } 209 } 210 211 auto H = dlopenInitialize(Path, Mode); 212 if (!H) { 213 DLFcnError = toString(H.takeError()); 214 return nullptr; 215 } 216 217 return *H; 218 } 219 220 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) { 221 runAtExits(DSOHandle); 222 return 0; 223 } 224 225 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) { 226 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); 227 if (!Addr) { 228 DLFcnError = toString(Addr.takeError()); 229 return 0; 230 } 231 232 return Addr->toPtr<void *>(); 233 } 234 235 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, 236 void *DSOHandle) { 237 // FIXME: Handle out-of-memory errors, returning -1 if OOM. 238 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 239 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); 240 assert(JDS && "JITDylib state not initialized"); 241 JDS->AtExits.push_back({F, Arg}); 242 return 0; 243 } 244 245 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) { 246 // FIXME: Should atexits be allowed to run concurrently with access to 247 // JDState? 248 AtExitsVector V; 249 { 250 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 251 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); 252 assert(JDS && "JITDlybi state not initialized"); 253 std::swap(V, JDS->AtExits); 254 } 255 256 while (!V.empty()) { 257 auto &AE = V.back(); 258 AE.Func(AE.Arg); 259 V.pop_back(); 260 } 261 } 262 263 Expected<std::pair<const char *, size_t>> 264 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { 265 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 266 auto I = ThreadDataSections.upper_bound(ThreadData); 267 // Check that we have a valid entry conovering this address. 268 if (I == ThreadDataSections.begin()) 269 return make_error<StringError>("No thread local data section for key"); 270 I = std::prev(I); 271 if (ThreadData >= I->first + I->second) 272 return make_error<StringError>("No thread local data section for key"); 273 return *I; 274 } 275 276 ELFNixPlatformRuntimeState::PerJITDylibState * 277 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { 278 auto I = JDStates.find(DSOHandle); 279 if (I == JDStates.end()) 280 return nullptr; 281 return &I->second; 282 } 283 284 ELFNixPlatformRuntimeState::PerJITDylibState * 285 ELFNixPlatformRuntimeState::getJITDylibStateByName(string_view Name) { 286 // FIXME: Avoid creating string copy here. 287 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); 288 if (I == JDNameToHeader.end()) 289 return nullptr; 290 void *H = I->second; 291 auto J = JDStates.find(H); 292 assert(J != JDStates.end() && 293 "JITDylib has name map entry but no header map entry"); 294 return &J->second; 295 } 296 297 ELFNixPlatformRuntimeState::PerJITDylibState & 298 ELFNixPlatformRuntimeState::getOrCreateJITDylibState( 299 ELFNixJITDylibInitializers &MOJDIs) { 300 void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>(); 301 302 auto &JDS = JDStates[Header]; 303 304 // If this entry hasn't been created yet. 305 if (!JDS.Header) { 306 assert(!JDNameToHeader.count(MOJDIs.Name) && 307 "JITDylib has header map entry but no name map entry"); 308 JDNameToHeader[MOJDIs.Name] = Header; 309 JDS.Header = Header; 310 } 311 312 return JDS; 313 } 314 315 Error ELFNixPlatformRuntimeState::registerThreadDataSection( 316 span<const char> ThreadDataSection) { 317 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 318 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); 319 if (I != ThreadDataSections.begin()) { 320 auto J = std::prev(I); 321 if (J->first + J->second > ThreadDataSection.data()) 322 return make_error<StringError>("Overlapping .tdata sections"); 323 } 324 ThreadDataSections.insert( 325 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); 326 return Error::success(); 327 } 328 329 Expected<ExecutorAddr> 330 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, 331 string_view Sym) { 332 Expected<ExecutorAddr> Result((ExecutorAddr())); 333 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( 334 SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag, 335 Result, 336 ExecutorAddr::fromPtr(DSOHandle), 337 Sym)) 338 return std::move(Err); 339 return Result; 340 } 341 342 Expected<ELFNixJITDylibInitializerSequence> 343 ELFNixPlatformRuntimeState::getJITDylibInitializersByName(string_view Path) { 344 Expected<ELFNixJITDylibInitializerSequence> Result( 345 (ELFNixJITDylibInitializerSequence())); 346 std::string PathStr(Path.data(), Path.size()); 347 if (auto Err = 348 WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>( 349 SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result, 350 Path)) 351 return std::move(Err); 352 return Result; 353 } 354 355 Expected<void *> ELFNixPlatformRuntimeState::dlopenInitialize(string_view Path, 356 int Mode) { 357 // Either our JITDylib wasn't loaded, or it or one of its dependencies allows 358 // reinitialization. We need to call in to the JIT to see if there's any new 359 // work pending. 360 auto InitSeq = getJITDylibInitializersByName(Path); 361 if (!InitSeq) 362 return InitSeq.takeError(); 363 364 // Init sequences should be non-empty. 365 if (InitSeq->empty()) 366 return make_error<StringError>( 367 "__orc_rt_elfnix_get_initializers returned an " 368 "empty init sequence"); 369 370 // Otherwise register and run initializers for each JITDylib. 371 for (auto &MOJDIs : *InitSeq) 372 if (auto Err = initializeJITDylib(MOJDIs)) 373 return std::move(Err); 374 375 // Return the header for the last item in the list. 376 auto *JDS = getJITDylibStateByHeaderAddr( 377 InitSeq->back().DSOHandleAddress.toPtr<void *>()); 378 assert(JDS && "Missing state entry for JD"); 379 return JDS->Header; 380 } 381 382 long getPriority(const std::string &name) { 383 auto pos = name.find_last_not_of("0123456789"); 384 if (pos == name.size() - 1) 385 return 65535; 386 else 387 return std::strtol(name.c_str() + pos + 1, nullptr, 10); 388 } 389 390 Error ELFNixPlatformRuntimeState::initializeJITDylib( 391 ELFNixJITDylibInitializers &MOJDIs) { 392 393 auto &JDS = getOrCreateJITDylibState(MOJDIs); 394 ++JDS.RefCount; 395 396 using SectionList = std::vector<ExecutorAddrRange>; 397 std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(), 398 [](const std::pair<std::string, SectionList> &LHS, 399 const std::pair<std::string, SectionList> &RHS) -> bool { 400 return getPriority(LHS.first) < getPriority(RHS.first); 401 }); 402 for (auto &Entry : MOJDIs.InitSections) 403 if (auto Err = runInitArray(Entry.second, MOJDIs)) 404 return Err; 405 406 return Error::success(); 407 } 408 class ELFNixPlatformRuntimeTLVManager { 409 public: 410 void *getInstance(const char *ThreadData); 411 412 private: 413 std::unordered_map<const char *, char *> Instances; 414 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; 415 }; 416 417 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { 418 auto I = Instances.find(ThreadData); 419 if (I != Instances.end()) 420 return I->second; 421 auto TDS = 422 ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); 423 if (!TDS) { 424 __orc_rt_log_error(toString(TDS.takeError()).c_str()); 425 return nullptr; 426 } 427 428 auto &Allocated = AllocatedSections[TDS->first]; 429 if (!Allocated) { 430 Allocated = std::make_unique<char[]>(TDS->second); 431 memcpy(Allocated.get(), TDS->first, TDS->second); 432 } 433 size_t ThreadDataDelta = ThreadData - TDS->first; 434 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); 435 436 char *Instance = Allocated.get() + ThreadDataDelta; 437 Instances[ThreadData] = Instance; 438 return Instance; 439 } 440 441 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) { 442 delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr); 443 } 444 445 } // end anonymous namespace 446 447 //------------------------------------------------------------------------------ 448 // JIT entry points 449 //------------------------------------------------------------------------------ 450 451 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 452 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) { 453 return WrapperFunction<void(uint64_t)>::handle( 454 ArgData, ArgSize, 455 [](uint64_t &DSOHandle) { 456 ELFNixPlatformRuntimeState::initialize( 457 reinterpret_cast<void *>(DSOHandle)); 458 }) 459 .release(); 460 } 461 462 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 463 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) { 464 ELFNixPlatformRuntimeState::destroy(); 465 return WrapperFunctionResult().release(); 466 } 467 468 /// Wrapper function for registering metadata on a per-object basis. 469 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 470 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) { 471 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: 472 handle(ArgData, ArgSize, 473 [](ELFNixPerObjectSectionsToRegister &POSR) { 474 return ELFNixPlatformRuntimeState::get().registerObjectSections( 475 std::move(POSR)); 476 }) 477 .release(); 478 } 479 480 /// Wrapper for releasing per-object metadat. 481 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 482 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) { 483 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: 484 handle(ArgData, ArgSize, 485 [](ELFNixPerObjectSectionsToRegister &POSR) { 486 return ELFNixPlatformRuntimeState::get() 487 .deregisterObjectSections(std::move(POSR)); 488 }) 489 .release(); 490 } 491 492 //------------------------------------------------------------------------------ 493 // TLV support 494 //------------------------------------------------------------------------------ 495 496 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) { 497 auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>( 498 pthread_getspecific(D->Key)); 499 if (!TLVMgr) 500 TLVMgr = new ELFNixPlatformRuntimeTLVManager(); 501 if (pthread_setspecific(D->Key, TLVMgr)) { 502 __orc_rt_log_error("Call to pthread_setspecific failed"); 503 return nullptr; 504 } 505 506 return TLVMgr->getInstance( 507 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); 508 } 509 510 ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl( 511 TLSDescriptor *D, const char *ThreadPointer) { 512 const char *TLVPtr = reinterpret_cast<const char *>( 513 __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry)); 514 return TLVPtr - ThreadPointer; 515 } 516 517 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 518 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) { 519 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( 520 ArgData, ArgSize, 521 []() -> Expected<uint64_t> { 522 pthread_key_t Key; 523 if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) { 524 __orc_rt_log_error("Call to pthread_key_create failed"); 525 return make_error<StringError>(strerror(Err)); 526 } 527 return static_cast<uint64_t>(Key); 528 }) 529 .release(); 530 } 531 532 //------------------------------------------------------------------------------ 533 // cxa_atexit support 534 //------------------------------------------------------------------------------ 535 536 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg, 537 void *dso_handle) { 538 return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg, 539 dso_handle); 540 } 541 542 int __orc_rt_elfnix_atexit(void (*func)(void *)) { 543 auto &PlatformRTState = ELFNixPlatformRuntimeState::get(); 544 return ELFNixPlatformRuntimeState::get().registerAtExit( 545 func, NULL, PlatformRTState.getPlatformJDDSOHandle()); 546 } 547 548 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) { 549 ELFNixPlatformRuntimeState::get().runAtExits(dso_handle); 550 } 551 552 //------------------------------------------------------------------------------ 553 // JIT'd dlfcn alternatives. 554 //------------------------------------------------------------------------------ 555 556 const char *__orc_rt_elfnix_jit_dlerror() { 557 return ELFNixPlatformRuntimeState::get().dlerror(); 558 } 559 560 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) { 561 return ELFNixPlatformRuntimeState::get().dlopen(path, mode); 562 } 563 564 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) { 565 return ELFNixPlatformRuntimeState::get().dlclose(dso_handle); 566 } 567 568 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) { 569 return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol); 570 } 571 572 //------------------------------------------------------------------------------ 573 // ELFNix Run Program 574 //------------------------------------------------------------------------------ 575 576 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program( 577 const char *JITDylibName, const char *EntrySymbolName, int argc, 578 char *argv[]) { 579 using MainTy = int (*)(int, char *[]); 580 581 void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName, 582 __orc_rt::elfnix::ORC_RT_RTLD_LAZY); 583 if (!H) { 584 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); 585 return -1; 586 } 587 588 auto *Main = 589 reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName)); 590 591 if (!Main) { 592 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); 593 return -1; 594 } 595 596 int Result = Main(argc, argv); 597 598 if (__orc_rt_elfnix_jit_dlclose(H) == -1) 599 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); 600 601 return Result; 602 } 603