1 //===- elfnix_platform.cpp ------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains code required to load the rest of the ELF-on-*IX runtime. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "elfnix_platform.h" 14 #include "common.h" 15 #include "error.h" 16 #include "wrapper_function_utils.h" 17 18 #include <algorithm> 19 #include <map> 20 #include <mutex> 21 #include <sstream> 22 #include <string_view> 23 #include <unordered_map> 24 #include <vector> 25 26 using namespace __orc_rt; 27 using namespace __orc_rt::elfnix; 28 29 // Declare function tags for functions in the JIT process. 30 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_initializers_tag) 31 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_get_deinitializers_tag) 32 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_elfnix_symbol_lookup_tag) 33 34 // eh-frame registration functions, made available via aliases 35 // installed by the Platform 36 extern "C" void __orc_rt_register_eh_frame_section(const void *); 37 extern "C" void __orc_rt_deregister_eh_frame_section(const void *); 38 39 namespace { 40 41 Error validatePointerSectionExtent(const char *SectionName, 42 const ExecutorAddrRange &SE) { 43 if (SE.size() % sizeof(uintptr_t)) { 44 std::ostringstream ErrMsg; 45 ErrMsg << std::hex << "Size of " << SectionName << " 0x" 46 << SE.Start.getValue() << " -- 0x" << SE.End.getValue() 47 << " is not a pointer multiple"; 48 return make_error<StringError>(ErrMsg.str()); 49 } 50 return Error::success(); 51 } 52 53 Error runInitArray(const std::vector<ExecutorAddrRange> &InitArraySections, 54 const ELFNixJITDylibInitializers &MOJDIs) { 55 56 for (const auto &ModInits : InitArraySections) { 57 if (auto Err = validatePointerSectionExtent(".init_array", ModInits)) 58 return Err; 59 60 using InitFunc = void (*)(); 61 for (auto *Init : ModInits.toSpan<InitFunc>()) 62 (*Init)(); 63 } 64 65 return Error::success(); 66 } 67 68 struct TLSInfoEntry { 69 unsigned long Key = 0; 70 unsigned long DataAddress = 0; 71 }; 72 73 struct TLSDescriptor { 74 void (*Resolver)(void *); 75 TLSInfoEntry *InfoEntry; 76 }; 77 78 class ELFNixPlatformRuntimeState { 79 private: 80 struct AtExitEntry { 81 void (*Func)(void *); 82 void *Arg; 83 }; 84 85 using AtExitsVector = std::vector<AtExitEntry>; 86 87 struct PerJITDylibState { 88 void *Header = nullptr; 89 size_t RefCount = 0; 90 bool AllowReinitialization = false; 91 AtExitsVector AtExits; 92 }; 93 94 public: 95 static void initialize(void *DSOHandle); 96 static ELFNixPlatformRuntimeState &get(); 97 static void destroy(); 98 99 ELFNixPlatformRuntimeState(void *DSOHandle) 100 : PlatformJDDSOHandle(DSOHandle) {} 101 102 // Delete copy and move constructors. 103 ELFNixPlatformRuntimeState(const ELFNixPlatformRuntimeState &) = delete; 104 ELFNixPlatformRuntimeState & 105 operator=(const ELFNixPlatformRuntimeState &) = delete; 106 ELFNixPlatformRuntimeState(ELFNixPlatformRuntimeState &&) = delete; 107 ELFNixPlatformRuntimeState &operator=(ELFNixPlatformRuntimeState &&) = delete; 108 109 Error registerObjectSections(ELFNixPerObjectSectionsToRegister POSR); 110 Error deregisterObjectSections(ELFNixPerObjectSectionsToRegister POSR); 111 112 const char *dlerror(); 113 void *dlopen(std::string_view Name, int Mode); 114 int dlclose(void *DSOHandle); 115 void *dlsym(void *DSOHandle, std::string_view Symbol); 116 117 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); 118 void runAtExits(void *DSOHandle); 119 120 /// Returns the base address of the section containing ThreadData. 121 Expected<std::pair<const char *, size_t>> 122 getThreadDataSectionFor(const char *ThreadData); 123 124 void *getPlatformJDDSOHandle() { return PlatformJDDSOHandle; } 125 126 private: 127 PerJITDylibState *getJITDylibStateByHeaderAddr(void *DSOHandle); 128 PerJITDylibState *getJITDylibStateByName(std::string_view Path); 129 PerJITDylibState & 130 getOrCreateJITDylibState(ELFNixJITDylibInitializers &MOJDIs); 131 132 Error registerThreadDataSection(span<const char> ThreadDataSection); 133 134 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, 135 std::string_view Symbol); 136 137 Expected<ELFNixJITDylibInitializerSequence> 138 getJITDylibInitializersByName(std::string_view Path); 139 Expected<void *> dlopenInitialize(std::string_view Path, int Mode); 140 Error initializeJITDylib(ELFNixJITDylibInitializers &MOJDIs); 141 142 static ELFNixPlatformRuntimeState *MOPS; 143 144 void *PlatformJDDSOHandle; 145 146 // FIXME: Move to thread-state. 147 std::string DLFcnError; 148 149 std::recursive_mutex JDStatesMutex; 150 std::unordered_map<void *, PerJITDylibState> JDStates; 151 std::unordered_map<std::string, void *> JDNameToHeader; 152 153 std::mutex ThreadDataSectionsMutex; 154 std::map<const char *, size_t> ThreadDataSections; 155 }; 156 157 ELFNixPlatformRuntimeState *ELFNixPlatformRuntimeState::MOPS = nullptr; 158 159 void ELFNixPlatformRuntimeState::initialize(void *DSOHandle) { 160 assert(!MOPS && "ELFNixPlatformRuntimeState should be null"); 161 MOPS = new ELFNixPlatformRuntimeState(DSOHandle); 162 } 163 164 ELFNixPlatformRuntimeState &ELFNixPlatformRuntimeState::get() { 165 assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); 166 return *MOPS; 167 } 168 169 void ELFNixPlatformRuntimeState::destroy() { 170 assert(MOPS && "ELFNixPlatformRuntimeState not initialized"); 171 delete MOPS; 172 } 173 174 Error ELFNixPlatformRuntimeState::registerObjectSections( 175 ELFNixPerObjectSectionsToRegister POSR) { 176 if (POSR.EHFrameSection.Start) 177 __orc_rt_register_eh_frame_section( 178 POSR.EHFrameSection.Start.toPtr<const char *>()); 179 180 if (POSR.ThreadDataSection.Start) { 181 if (auto Err = registerThreadDataSection( 182 POSR.ThreadDataSection.toSpan<const char>())) 183 return Err; 184 } 185 186 return Error::success(); 187 } 188 189 Error ELFNixPlatformRuntimeState::deregisterObjectSections( 190 ELFNixPerObjectSectionsToRegister POSR) { 191 if (POSR.EHFrameSection.Start) 192 __orc_rt_deregister_eh_frame_section( 193 POSR.EHFrameSection.Start.toPtr<const char *>()); 194 195 return Error::success(); 196 } 197 198 const char *ELFNixPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } 199 200 void *ELFNixPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { 201 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 202 203 // Use fast path if all JITDylibs are already loaded and don't require 204 // re-running initializers. 205 if (auto *JDS = getJITDylibStateByName(Path)) { 206 if (!JDS->AllowReinitialization) { 207 ++JDS->RefCount; 208 return JDS->Header; 209 } 210 } 211 212 auto H = dlopenInitialize(Path, Mode); 213 if (!H) { 214 DLFcnError = toString(H.takeError()); 215 return nullptr; 216 } 217 218 return *H; 219 } 220 221 int ELFNixPlatformRuntimeState::dlclose(void *DSOHandle) { 222 runAtExits(DSOHandle); 223 return 0; 224 } 225 226 void *ELFNixPlatformRuntimeState::dlsym(void *DSOHandle, 227 std::string_view Symbol) { 228 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); 229 if (!Addr) { 230 DLFcnError = toString(Addr.takeError()); 231 return 0; 232 } 233 234 return Addr->toPtr<void *>(); 235 } 236 237 int ELFNixPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, 238 void *DSOHandle) { 239 // FIXME: Handle out-of-memory errors, returning -1 if OOM. 240 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 241 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); 242 assert(JDS && "JITDylib state not initialized"); 243 JDS->AtExits.push_back({F, Arg}); 244 return 0; 245 } 246 247 void ELFNixPlatformRuntimeState::runAtExits(void *DSOHandle) { 248 // FIXME: Should atexits be allowed to run concurrently with access to 249 // JDState? 250 AtExitsVector V; 251 { 252 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 253 auto *JDS = getJITDylibStateByHeaderAddr(DSOHandle); 254 assert(JDS && "JITDlybi state not initialized"); 255 std::swap(V, JDS->AtExits); 256 } 257 258 while (!V.empty()) { 259 auto &AE = V.back(); 260 AE.Func(AE.Arg); 261 V.pop_back(); 262 } 263 } 264 265 Expected<std::pair<const char *, size_t>> 266 ELFNixPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { 267 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 268 auto I = ThreadDataSections.upper_bound(ThreadData); 269 // Check that we have a valid entry conovering this address. 270 if (I == ThreadDataSections.begin()) 271 return make_error<StringError>("No thread local data section for key"); 272 I = std::prev(I); 273 if (ThreadData >= I->first + I->second) 274 return make_error<StringError>("No thread local data section for key"); 275 return *I; 276 } 277 278 ELFNixPlatformRuntimeState::PerJITDylibState * 279 ELFNixPlatformRuntimeState::getJITDylibStateByHeaderAddr(void *DSOHandle) { 280 auto I = JDStates.find(DSOHandle); 281 if (I == JDStates.end()) 282 return nullptr; 283 return &I->second; 284 } 285 286 ELFNixPlatformRuntimeState::PerJITDylibState * 287 ELFNixPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { 288 // FIXME: Avoid creating string copy here. 289 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); 290 if (I == JDNameToHeader.end()) 291 return nullptr; 292 void *H = I->second; 293 auto J = JDStates.find(H); 294 assert(J != JDStates.end() && 295 "JITDylib has name map entry but no header map entry"); 296 return &J->second; 297 } 298 299 ELFNixPlatformRuntimeState::PerJITDylibState & 300 ELFNixPlatformRuntimeState::getOrCreateJITDylibState( 301 ELFNixJITDylibInitializers &MOJDIs) { 302 void *Header = MOJDIs.DSOHandleAddress.toPtr<void *>(); 303 304 auto &JDS = JDStates[Header]; 305 306 // If this entry hasn't been created yet. 307 if (!JDS.Header) { 308 assert(!JDNameToHeader.count(MOJDIs.Name) && 309 "JITDylib has header map entry but no name map entry"); 310 JDNameToHeader[MOJDIs.Name] = Header; 311 JDS.Header = Header; 312 } 313 314 return JDS; 315 } 316 317 Error ELFNixPlatformRuntimeState::registerThreadDataSection( 318 span<const char> ThreadDataSection) { 319 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 320 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); 321 if (I != ThreadDataSections.begin()) { 322 auto J = std::prev(I); 323 if (J->first + J->second > ThreadDataSection.data()) 324 return make_error<StringError>("Overlapping .tdata sections"); 325 } 326 ThreadDataSections.insert( 327 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); 328 return Error::success(); 329 } 330 331 Expected<ExecutorAddr> 332 ELFNixPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, 333 std::string_view Sym) { 334 Expected<ExecutorAddr> Result((ExecutorAddr())); 335 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( 336 SPSExecutorAddr, SPSString)>::call(&__orc_rt_elfnix_symbol_lookup_tag, 337 Result, 338 ExecutorAddr::fromPtr(DSOHandle), 339 Sym)) 340 return std::move(Err); 341 return Result; 342 } 343 344 Expected<ELFNixJITDylibInitializerSequence> 345 ELFNixPlatformRuntimeState::getJITDylibInitializersByName( 346 std::string_view Path) { 347 Expected<ELFNixJITDylibInitializerSequence> Result( 348 (ELFNixJITDylibInitializerSequence())); 349 std::string PathStr(Path.data(), Path.size()); 350 if (auto Err = 351 WrapperFunction<SPSExpected<SPSELFNixJITDylibInitializerSequence>( 352 SPSString)>::call(&__orc_rt_elfnix_get_initializers_tag, Result, 353 Path)) 354 return std::move(Err); 355 return Result; 356 } 357 358 Expected<void *> 359 ELFNixPlatformRuntimeState::dlopenInitialize(std::string_view Path, int Mode) { 360 // Either our JITDylib wasn't loaded, or it or one of its dependencies allows 361 // reinitialization. We need to call in to the JIT to see if there's any new 362 // work pending. 363 auto InitSeq = getJITDylibInitializersByName(Path); 364 if (!InitSeq) 365 return InitSeq.takeError(); 366 367 // Init sequences should be non-empty. 368 if (InitSeq->empty()) 369 return make_error<StringError>( 370 "__orc_rt_elfnix_get_initializers returned an " 371 "empty init sequence"); 372 373 // Otherwise register and run initializers for each JITDylib. 374 for (auto &MOJDIs : *InitSeq) 375 if (auto Err = initializeJITDylib(MOJDIs)) 376 return std::move(Err); 377 378 // Return the header for the last item in the list. 379 auto *JDS = getJITDylibStateByHeaderAddr( 380 InitSeq->back().DSOHandleAddress.toPtr<void *>()); 381 assert(JDS && "Missing state entry for JD"); 382 return JDS->Header; 383 } 384 385 long getPriority(const std::string &name) { 386 auto pos = name.find_last_not_of("0123456789"); 387 if (pos == name.size() - 1) 388 return 65535; 389 else 390 return std::strtol(name.c_str() + pos + 1, nullptr, 10); 391 } 392 393 Error ELFNixPlatformRuntimeState::initializeJITDylib( 394 ELFNixJITDylibInitializers &MOJDIs) { 395 396 auto &JDS = getOrCreateJITDylibState(MOJDIs); 397 ++JDS.RefCount; 398 399 using SectionList = std::vector<ExecutorAddrRange>; 400 std::sort(MOJDIs.InitSections.begin(), MOJDIs.InitSections.end(), 401 [](const std::pair<std::string, SectionList> &LHS, 402 const std::pair<std::string, SectionList> &RHS) -> bool { 403 return getPriority(LHS.first) < getPriority(RHS.first); 404 }); 405 for (auto &Entry : MOJDIs.InitSections) 406 if (auto Err = runInitArray(Entry.second, MOJDIs)) 407 return Err; 408 409 return Error::success(); 410 } 411 class ELFNixPlatformRuntimeTLVManager { 412 public: 413 void *getInstance(const char *ThreadData); 414 415 private: 416 std::unordered_map<const char *, char *> Instances; 417 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; 418 }; 419 420 void *ELFNixPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { 421 auto I = Instances.find(ThreadData); 422 if (I != Instances.end()) 423 return I->second; 424 auto TDS = 425 ELFNixPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); 426 if (!TDS) { 427 __orc_rt_log_error(toString(TDS.takeError()).c_str()); 428 return nullptr; 429 } 430 431 auto &Allocated = AllocatedSections[TDS->first]; 432 if (!Allocated) { 433 Allocated = std::make_unique<char[]>(TDS->second); 434 memcpy(Allocated.get(), TDS->first, TDS->second); 435 } 436 size_t ThreadDataDelta = ThreadData - TDS->first; 437 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); 438 439 char *Instance = Allocated.get() + ThreadDataDelta; 440 Instances[ThreadData] = Instance; 441 return Instance; 442 } 443 444 void destroyELFNixTLVMgr(void *ELFNixTLVMgr) { 445 delete static_cast<ELFNixPlatformRuntimeTLVManager *>(ELFNixTLVMgr); 446 } 447 448 } // end anonymous namespace 449 450 //------------------------------------------------------------------------------ 451 // JIT entry points 452 //------------------------------------------------------------------------------ 453 454 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 455 __orc_rt_elfnix_platform_bootstrap(char *ArgData, size_t ArgSize) { 456 return WrapperFunction<void(uint64_t)>::handle( 457 ArgData, ArgSize, 458 [](uint64_t &DSOHandle) { 459 ELFNixPlatformRuntimeState::initialize( 460 reinterpret_cast<void *>(DSOHandle)); 461 }) 462 .release(); 463 } 464 465 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 466 __orc_rt_elfnix_platform_shutdown(char *ArgData, size_t ArgSize) { 467 ELFNixPlatformRuntimeState::destroy(); 468 return WrapperFunctionResult().release(); 469 } 470 471 /// Wrapper function for registering metadata on a per-object basis. 472 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 473 __orc_rt_elfnix_register_object_sections(char *ArgData, size_t ArgSize) { 474 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: 475 handle(ArgData, ArgSize, 476 [](ELFNixPerObjectSectionsToRegister &POSR) { 477 return ELFNixPlatformRuntimeState::get().registerObjectSections( 478 std::move(POSR)); 479 }) 480 .release(); 481 } 482 483 /// Wrapper for releasing per-object metadat. 484 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 485 __orc_rt_elfnix_deregister_object_sections(char *ArgData, size_t ArgSize) { 486 return WrapperFunction<SPSError(SPSELFNixPerObjectSectionsToRegister)>:: 487 handle(ArgData, ArgSize, 488 [](ELFNixPerObjectSectionsToRegister &POSR) { 489 return ELFNixPlatformRuntimeState::get() 490 .deregisterObjectSections(std::move(POSR)); 491 }) 492 .release(); 493 } 494 495 //------------------------------------------------------------------------------ 496 // TLV support 497 //------------------------------------------------------------------------------ 498 499 ORC_RT_INTERFACE void *__orc_rt_elfnix_tls_get_addr_impl(TLSInfoEntry *D) { 500 auto *TLVMgr = static_cast<ELFNixPlatformRuntimeTLVManager *>( 501 pthread_getspecific(D->Key)); 502 if (!TLVMgr) 503 TLVMgr = new ELFNixPlatformRuntimeTLVManager(); 504 if (pthread_setspecific(D->Key, TLVMgr)) { 505 __orc_rt_log_error("Call to pthread_setspecific failed"); 506 return nullptr; 507 } 508 509 return TLVMgr->getInstance( 510 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); 511 } 512 513 ORC_RT_INTERFACE ptrdiff_t ___orc_rt_elfnix_tlsdesc_resolver_impl( 514 TLSDescriptor *D, const char *ThreadPointer) { 515 const char *TLVPtr = reinterpret_cast<const char *>( 516 __orc_rt_elfnix_tls_get_addr_impl(D->InfoEntry)); 517 return TLVPtr - ThreadPointer; 518 } 519 520 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 521 __orc_rt_elfnix_create_pthread_key(char *ArgData, size_t ArgSize) { 522 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( 523 ArgData, ArgSize, 524 []() -> Expected<uint64_t> { 525 pthread_key_t Key; 526 if (int Err = pthread_key_create(&Key, destroyELFNixTLVMgr)) { 527 __orc_rt_log_error("Call to pthread_key_create failed"); 528 return make_error<StringError>(strerror(Err)); 529 } 530 return static_cast<uint64_t>(Key); 531 }) 532 .release(); 533 } 534 535 //------------------------------------------------------------------------------ 536 // cxa_atexit support 537 //------------------------------------------------------------------------------ 538 539 int __orc_rt_elfnix_cxa_atexit(void (*func)(void *), void *arg, 540 void *dso_handle) { 541 return ELFNixPlatformRuntimeState::get().registerAtExit(func, arg, 542 dso_handle); 543 } 544 545 int __orc_rt_elfnix_atexit(void (*func)(void *)) { 546 auto &PlatformRTState = ELFNixPlatformRuntimeState::get(); 547 return ELFNixPlatformRuntimeState::get().registerAtExit( 548 func, NULL, PlatformRTState.getPlatformJDDSOHandle()); 549 } 550 551 void __orc_rt_elfnix_cxa_finalize(void *dso_handle) { 552 ELFNixPlatformRuntimeState::get().runAtExits(dso_handle); 553 } 554 555 //------------------------------------------------------------------------------ 556 // JIT'd dlfcn alternatives. 557 //------------------------------------------------------------------------------ 558 559 const char *__orc_rt_elfnix_jit_dlerror() { 560 return ELFNixPlatformRuntimeState::get().dlerror(); 561 } 562 563 void *__orc_rt_elfnix_jit_dlopen(const char *path, int mode) { 564 return ELFNixPlatformRuntimeState::get().dlopen(path, mode); 565 } 566 567 int __orc_rt_elfnix_jit_dlclose(void *dso_handle) { 568 return ELFNixPlatformRuntimeState::get().dlclose(dso_handle); 569 } 570 571 void *__orc_rt_elfnix_jit_dlsym(void *dso_handle, const char *symbol) { 572 return ELFNixPlatformRuntimeState::get().dlsym(dso_handle, symbol); 573 } 574 575 //------------------------------------------------------------------------------ 576 // ELFNix Run Program 577 //------------------------------------------------------------------------------ 578 579 ORC_RT_INTERFACE int64_t __orc_rt_elfnix_run_program( 580 const char *JITDylibName, const char *EntrySymbolName, int argc, 581 char *argv[]) { 582 using MainTy = int (*)(int, char *[]); 583 584 void *H = __orc_rt_elfnix_jit_dlopen(JITDylibName, 585 __orc_rt::elfnix::ORC_RT_RTLD_LAZY); 586 if (!H) { 587 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); 588 return -1; 589 } 590 591 auto *Main = 592 reinterpret_cast<MainTy>(__orc_rt_elfnix_jit_dlsym(H, EntrySymbolName)); 593 594 if (!Main) { 595 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); 596 return -1; 597 } 598 599 int Result = Main(argc, argv); 600 601 if (__orc_rt_elfnix_jit_dlclose(H) == -1) 602 __orc_rt_log_error(__orc_rt_elfnix_jit_dlerror()); 603 604 return Result; 605 } 606