1 //===- macho_platform.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains code required to load the rest of the MachO runtime. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "macho_platform.h" 14 #include "common.h" 15 #include "debug.h" 16 #include "error.h" 17 #include "interval_map.h" 18 #include "wrapper_function_utils.h" 19 20 #include <algorithm> 21 #include <ios> 22 #include <map> 23 #include <mutex> 24 #include <sstream> 25 #include <string_view> 26 #include <unordered_map> 27 #include <unordered_set> 28 #include <vector> 29 30 #define DEBUG_TYPE "macho_platform" 31 32 using namespace __orc_rt; 33 using namespace __orc_rt::macho; 34 35 // Declare function tags for functions in the JIT process. 36 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag) 37 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag) 38 39 // Objective-C types. 40 struct objc_class; 41 struct objc_image_info; 42 struct objc_object; 43 struct objc_selector; 44 45 using Class = objc_class *; 46 using id = objc_object *; 47 using SEL = objc_selector *; 48 49 // Objective-C registration functions. 50 // These are weakly imported. If the Objective-C runtime has not been loaded 51 // then code containing Objective-C sections will generate an error. 52 extern "C" id objc_msgSend(id, SEL, ...) ORC_RT_WEAK_IMPORT; 53 extern "C" Class objc_readClassPair(Class, 54 const objc_image_info *) ORC_RT_WEAK_IMPORT; 55 extern "C" SEL sel_registerName(const char *) ORC_RT_WEAK_IMPORT; 56 57 // Swift types. 58 class ProtocolRecord; 59 class ProtocolConformanceRecord; 60 class TypeMetadataRecord; 61 62 extern "C" void 63 swift_registerProtocols(const ProtocolRecord *begin, 64 const ProtocolRecord *end) ORC_RT_WEAK_IMPORT; 65 66 extern "C" void swift_registerProtocolConformances( 67 const ProtocolConformanceRecord *begin, 68 const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT; 69 70 extern "C" void swift_registerTypeMetadataRecords( 71 const TypeMetadataRecord *begin, 72 const TypeMetadataRecord *end) ORC_RT_WEAK_IMPORT; 73 74 // Libunwind prototypes. 75 struct unw_dynamic_unwind_sections { 76 uintptr_t dso_base; 77 uintptr_t dwarf_section; 78 size_t dwarf_section_length; 79 uintptr_t compact_unwind_section; 80 size_t compact_unwind_section_length; 81 }; 82 83 typedef int (*unw_find_dynamic_unwind_sections)( 84 uintptr_t addr, struct unw_dynamic_unwind_sections *info); 85 86 extern "C" int __unw_add_find_dynamic_unwind_sections( 87 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections) 88 ORC_RT_WEAK_IMPORT; 89 90 extern "C" int __unw_remove_find_dynamic_unwind_sections( 91 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections) 92 ORC_RT_WEAK_IMPORT; 93 94 namespace { 95 96 struct MachOJITDylibDepInfo { 97 bool Sealed = false; 98 std::vector<ExecutorAddr> DepHeaders; 99 }; 100 101 using MachOJITDylibDepInfoMap = 102 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>; 103 104 } // anonymous namespace 105 106 namespace __orc_rt { 107 108 using SPSMachOObjectPlatformSectionsMap = 109 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; 110 111 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>; 112 113 using SPSMachOJITDylibDepInfoMap = 114 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>; 115 116 template <> 117 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> { 118 public: 119 static size_t size(const MachOJITDylibDepInfo &JDI) { 120 return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders); 121 } 122 123 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) { 124 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed, 125 JDI.DepHeaders); 126 } 127 128 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) { 129 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed, 130 JDI.DepHeaders); 131 } 132 }; 133 134 struct UnwindSectionInfo { 135 std::vector<ExecutorAddrRange> CodeRanges; 136 ExecutorAddrRange DwarfSection; 137 ExecutorAddrRange CompactUnwindSection; 138 }; 139 140 using SPSUnwindSectionInfo = 141 SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange, 142 SPSExecutorAddrRange>; 143 144 template <> 145 class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> { 146 public: 147 static size_t size(const UnwindSectionInfo &USI) { 148 return SPSUnwindSectionInfo::AsArgList::size( 149 USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); 150 } 151 152 static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) { 153 return SPSUnwindSectionInfo::AsArgList::serialize( 154 OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); 155 } 156 157 static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) { 158 return SPSUnwindSectionInfo::AsArgList::deserialize( 159 IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection); 160 } 161 }; 162 163 } // namespace __orc_rt 164 165 namespace { 166 struct TLVDescriptor { 167 void *(*Thunk)(TLVDescriptor *) = nullptr; 168 unsigned long Key = 0; 169 unsigned long DataAddress = 0; 170 }; 171 172 class MachOPlatformRuntimeState { 173 private: 174 struct AtExitEntry { 175 void (*Func)(void *); 176 void *Arg; 177 }; 178 179 using AtExitsVector = std::vector<AtExitEntry>; 180 181 /// Used to manage sections of fixed-sized metadata records (e.g. pointer 182 /// sections, selector refs, etc.) 183 template <typename RecordElement> class RecordSectionsTracker { 184 public: 185 /// Add a section to the "new" list. 186 void add(span<RecordElement> Sec) { New.push_back(std::move(Sec)); } 187 188 /// Returns true if there are new sections to process. 189 bool hasNewSections() const { return !New.empty(); } 190 191 /// Returns the number of new sections to process. 192 size_t numNewSections() const { return New.size(); } 193 194 /// Process all new sections. 195 template <typename ProcessSectionFunc> 196 std::enable_if_t<std::is_void_v< 197 std::invoke_result_t<ProcessSectionFunc, span<RecordElement>>>> 198 processNewSections(ProcessSectionFunc &&ProcessSection) { 199 for (auto &Sec : New) 200 ProcessSection(Sec); 201 moveNewToProcessed(); 202 } 203 204 /// Proces all new sections with a fallible handler. 205 /// 206 /// Successfully handled sections will be moved to the Processed 207 /// list. 208 template <typename ProcessSectionFunc> 209 std::enable_if_t< 210 std::is_same_v<Error, std::invoke_result_t<ProcessSectionFunc, 211 span<RecordElement>>>, 212 Error> 213 processNewSections(ProcessSectionFunc &&ProcessSection) { 214 for (size_t I = 0; I != New.size(); ++I) { 215 if (auto Err = ProcessSection(New[I])) { 216 for (size_t J = 0; J != I; ++J) 217 Processed.push_back(New[J]); 218 New.erase(New.begin(), New.begin() + I); 219 return Err; 220 } 221 } 222 moveNewToProcessed(); 223 return Error::success(); 224 } 225 226 /// Move all sections back to New for reprocessing. 227 void reset() { 228 moveNewToProcessed(); 229 New = std::move(Processed); 230 } 231 232 /// Remove the section with the given range. 233 bool removeIfPresent(ExecutorAddrRange R) { 234 if (removeIfPresent(New, R)) 235 return true; 236 return removeIfPresent(Processed, R); 237 } 238 239 private: 240 void moveNewToProcessed() { 241 if (Processed.empty()) 242 Processed = std::move(New); 243 else { 244 Processed.reserve(Processed.size() + New.size()); 245 std::copy(New.begin(), New.end(), std::back_inserter(Processed)); 246 New.clear(); 247 } 248 } 249 250 bool removeIfPresent(std::vector<span<RecordElement>> &V, 251 ExecutorAddrRange R) { 252 auto RI = std::find_if( 253 V.rbegin(), V.rend(), 254 [RS = R.toSpan<RecordElement>()](const span<RecordElement> &E) { 255 return E.data() == RS.data(); 256 }); 257 if (RI != V.rend()) { 258 V.erase(std::next(RI).base()); 259 return true; 260 } 261 return false; 262 } 263 264 std::vector<span<RecordElement>> Processed; 265 std::vector<span<RecordElement>> New; 266 }; 267 268 struct UnwindSections { 269 UnwindSections(const UnwindSectionInfo &USI) 270 : DwarfSection(USI.DwarfSection.toSpan<char>()), 271 CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {} 272 273 span<char> DwarfSection; 274 span<char> CompactUnwindSection; 275 }; 276 277 using UnwindSectionsMap = 278 IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>; 279 280 struct JITDylibState { 281 std::string Name; 282 void *Header = nullptr; 283 bool Sealed = false; 284 size_t LinkedAgainstRefCount = 0; 285 size_t DlRefCount = 0; 286 std::vector<JITDylibState *> Deps; 287 AtExitsVector AtExits; 288 const objc_image_info *ObjCImageInfo = nullptr; 289 std::unordered_map<void *, std::vector<char>> DataSectionContent; 290 std::unordered_map<void *, size_t> ZeroInitRanges; 291 UnwindSectionsMap UnwindSections; 292 RecordSectionsTracker<void (*)()> ModInitsSections; 293 RecordSectionsTracker<void *> ObjCClassListSections; 294 RecordSectionsTracker<void *> ObjCSelRefsSections; 295 RecordSectionsTracker<char> Swift5ProtocolsSections; 296 RecordSectionsTracker<char> Swift5ProtocolConformancesSections; 297 RecordSectionsTracker<char> Swift5TypesSections; 298 299 bool referenced() const { 300 return LinkedAgainstRefCount != 0 || DlRefCount != 0; 301 } 302 }; 303 304 public: 305 static Error create(); 306 static MachOPlatformRuntimeState &get(); 307 static Error destroy(); 308 309 MachOPlatformRuntimeState() = default; 310 311 // Delete copy and move constructors. 312 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete; 313 MachOPlatformRuntimeState & 314 operator=(const MachOPlatformRuntimeState &) = delete; 315 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; 316 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; 317 318 Error initialize(); 319 Error shutdown(); 320 321 Error registerJITDylib(std::string Name, void *Header); 322 Error deregisterJITDylib(void *Header); 323 Error registerThreadDataSection(span<const char> ThreadDataSection); 324 Error deregisterThreadDataSection(span<const char> ThreadDataSection); 325 Error registerObjectPlatformSections( 326 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections, 327 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs); 328 Error deregisterObjectPlatformSections( 329 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections, 330 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs); 331 332 const char *dlerror(); 333 void *dlopen(std::string_view Name, int Mode); 334 int dlclose(void *DSOHandle); 335 void *dlsym(void *DSOHandle, std::string_view Symbol); 336 337 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); 338 void runAtExits(std::unique_lock<std::mutex> &JDStatesLock, 339 JITDylibState &JDS); 340 void runAtExits(void *DSOHandle); 341 342 /// Returns the base address of the section containing ThreadData. 343 Expected<std::pair<const char *, size_t>> 344 getThreadDataSectionFor(const char *ThreadData); 345 346 private: 347 JITDylibState *getJITDylibStateByHeader(void *DSOHandle); 348 JITDylibState *getJITDylibStateByName(std::string_view Path); 349 350 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, 351 std::string_view Symbol); 352 353 bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info); 354 355 static int findDynamicUnwindSections(uintptr_t addr, 356 unw_dynamic_unwind_sections *info); 357 static Error registerEHFrames(span<const char> EHFrameSection); 358 static Error deregisterEHFrames(span<const char> EHFrameSection); 359 360 static Error registerObjCSelectors(JITDylibState &JDS); 361 static Error registerObjCClasses(JITDylibState &JDS); 362 static Error registerSwift5Protocols(JITDylibState &JDS); 363 static Error registerSwift5ProtocolConformances(JITDylibState &JDS); 364 static Error registerSwift5Types(JITDylibState &JDS); 365 static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock, 366 JITDylibState &JDS); 367 368 Expected<void *> dlopenImpl(std::string_view Path, int Mode); 369 Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock, 370 JITDylibState &JDS); 371 Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock, 372 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo); 373 374 Error dlcloseImpl(void *DSOHandle); 375 Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock, 376 JITDylibState &JDS); 377 378 static MachOPlatformRuntimeState *MOPS; 379 380 bool UseCallbackStyleUnwindInfo = false; 381 382 // FIXME: Move to thread-state. 383 std::string DLFcnError; 384 385 // APIMutex guards against concurrent entry into key "dyld" API functions 386 // (e.g. dlopen, dlclose). 387 std::recursive_mutex DyldAPIMutex; 388 389 // JDStatesMutex guards the data structures that hold JITDylib state. 390 std::mutex JDStatesMutex; 391 std::unordered_map<void *, JITDylibState> JDStates; 392 std::unordered_map<std::string_view, void *> JDNameToHeader; 393 394 // ThreadDataSectionsMutex guards thread local data section state. 395 std::mutex ThreadDataSectionsMutex; 396 std::map<const char *, size_t> ThreadDataSections; 397 }; 398 399 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr; 400 401 Error MachOPlatformRuntimeState::create() { 402 assert(!MOPS && "MachOPlatformRuntimeState should be null"); 403 MOPS = new MachOPlatformRuntimeState(); 404 return MOPS->initialize(); 405 } 406 407 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { 408 assert(MOPS && "MachOPlatformRuntimeState not initialized"); 409 return *MOPS; 410 } 411 412 Error MachOPlatformRuntimeState::destroy() { 413 assert(MOPS && "MachOPlatformRuntimeState not initialized"); 414 auto Err = MOPS->shutdown(); 415 delete MOPS; 416 return Err; 417 } 418 419 Error MachOPlatformRuntimeState::initialize() { 420 UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections && 421 __unw_remove_find_dynamic_unwind_sections; 422 if (UseCallbackStyleUnwindInfo) { 423 ORC_RT_DEBUG({ 424 printdbg("__unw_add/remove_find_dynamic_unwind_sections available." 425 " Using callback-based frame info lookup.\n"); 426 }); 427 if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections)) 428 return make_error<StringError>( 429 "Could not register findDynamicUnwindSections"); 430 } else { 431 ORC_RT_DEBUG({ 432 printdbg("__unw_add/remove_find_dynamic_unwind_sections not available." 433 " Using classic frame info registration.\n"); 434 }); 435 } 436 return Error::success(); 437 } 438 439 Error MachOPlatformRuntimeState::shutdown() { 440 if (__unw_add_find_dynamic_unwind_sections && 441 __unw_remove_find_dynamic_unwind_sections) { 442 if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) { 443 ORC_RT_DEBUG( 444 { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); }); 445 } 446 } 447 return Error::success(); 448 } 449 450 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name, 451 void *Header) { 452 ORC_RT_DEBUG({ 453 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); 454 }); 455 std::lock_guard<std::mutex> Lock(JDStatesMutex); 456 if (JDStates.count(Header)) { 457 std::ostringstream ErrStream; 458 ErrStream << "Duplicate JITDylib registration for header " << Header 459 << " (name = " << Name << ")"; 460 return make_error<StringError>(ErrStream.str()); 461 } 462 if (JDNameToHeader.count(Name)) { 463 std::ostringstream ErrStream; 464 ErrStream << "Duplicate JITDylib registration for header " << Header 465 << " (header = " << Header << ")"; 466 return make_error<StringError>(ErrStream.str()); 467 } 468 469 auto &JDS = JDStates[Header]; 470 JDS.Name = std::move(Name); 471 JDS.Header = Header; 472 JDNameToHeader[JDS.Name] = Header; 473 return Error::success(); 474 } 475 476 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) { 477 std::lock_guard<std::mutex> Lock(JDStatesMutex); 478 auto I = JDStates.find(Header); 479 if (I == JDStates.end()) { 480 std::ostringstream ErrStream; 481 ErrStream << "Attempted to deregister unrecognized header " << Header; 482 return make_error<StringError>(ErrStream.str()); 483 } 484 485 // Remove std::string construction once we can use C++20. 486 auto J = JDNameToHeader.find( 487 std::string(I->second.Name.data(), I->second.Name.size())); 488 assert(J != JDNameToHeader.end() && 489 "Missing JDNameToHeader entry for JITDylib"); 490 491 ORC_RT_DEBUG({ 492 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), 493 Header); 494 }); 495 496 JDNameToHeader.erase(J); 497 JDStates.erase(I); 498 return Error::success(); 499 } 500 501 Error MachOPlatformRuntimeState::registerThreadDataSection( 502 span<const char> ThreadDataSection) { 503 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 504 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); 505 if (I != ThreadDataSections.begin()) { 506 auto J = std::prev(I); 507 if (J->first + J->second > ThreadDataSection.data()) 508 return make_error<StringError>("Overlapping __thread_data sections"); 509 } 510 ThreadDataSections.insert( 511 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); 512 return Error::success(); 513 } 514 515 Error MachOPlatformRuntimeState::deregisterThreadDataSection( 516 span<const char> ThreadDataSection) { 517 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 518 auto I = ThreadDataSections.find(ThreadDataSection.data()); 519 if (I == ThreadDataSections.end()) 520 return make_error<StringError>("Attempt to deregister unknown thread data " 521 "section"); 522 ThreadDataSections.erase(I); 523 return Error::success(); 524 } 525 526 Error MachOPlatformRuntimeState::registerObjectPlatformSections( 527 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo, 528 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) { 529 530 // FIXME: Reject platform section registration after the JITDylib is 531 // sealed? 532 533 ORC_RT_DEBUG({ 534 printdbg("MachOPlatform: Registering object sections for %p.\n", 535 HeaderAddr.toPtr<void *>()); 536 }); 537 538 std::lock_guard<std::mutex> Lock(JDStatesMutex); 539 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); 540 if (!JDS) { 541 std::ostringstream ErrStream; 542 ErrStream << "Could not register object platform sections for " 543 "unrecognized header " 544 << HeaderAddr.toPtr<void *>(); 545 return make_error<StringError>(ErrStream.str()); 546 } 547 548 if (UnwindInfo && UseCallbackStyleUnwindInfo) { 549 ORC_RT_DEBUG({ 550 printdbg(" Registering new-style unwind info for:\n" 551 " DWARF: %p -- %p\n" 552 " Compact-unwind: %p -- %p\n" 553 " for:\n", 554 UnwindInfo->DwarfSection.Start.toPtr<void *>(), 555 UnwindInfo->DwarfSection.End.toPtr<void *>(), 556 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(), 557 UnwindInfo->CompactUnwindSection.End.toPtr<void *>()); 558 }); 559 for (auto &CodeRange : UnwindInfo->CodeRanges) { 560 JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(), 561 CodeRange.End.toPtr<char *>(), *UnwindInfo); 562 ORC_RT_DEBUG({ 563 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(), 564 CodeRange.End.toPtr<void *>()); 565 }); 566 } 567 } 568 569 for (auto &KV : Secs) { 570 // FIXME: Validate section ranges? 571 if (KV.first == "__TEXT,__eh_frame") { 572 if (!UseCallbackStyleUnwindInfo) { 573 // Use classic libunwind registration. 574 if (auto Err = registerEHFrames(KV.second.toSpan<const char>())) 575 return Err; 576 } 577 } else if (KV.first == "__DATA,__data") { 578 assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) && 579 "Address already registered."); 580 auto S = KV.second.toSpan<char>(); 581 JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] = 582 std::vector<char>(S.begin(), S.end()); 583 } else if (KV.first == "__DATA,__common") { 584 // fprintf(stderr, "Adding zero-init range %llx -- %llx\n", 585 // KV.second.Start.getValue(), KV.second.size()); 586 JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size(); 587 } else if (KV.first == "__DATA,__thread_data") { 588 if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>())) 589 return Err; 590 } else if (KV.first == "__DATA,__objc_selrefs") 591 JDS->ObjCSelRefsSections.add(KV.second.toSpan<void *>()); 592 else if (KV.first == "__DATA,__objc_classlist") 593 JDS->ObjCClassListSections.add(KV.second.toSpan<void *>()); 594 else if (KV.first == "__TEXT,__swift5_protos") 595 JDS->Swift5ProtocolsSections.add(KV.second.toSpan<char>()); 596 else if (KV.first == "__TEXT,__swift5_proto") 597 JDS->Swift5ProtocolConformancesSections.add(KV.second.toSpan<char>()); 598 else if (KV.first == "__TEXT,__swift5_types") 599 JDS->Swift5TypesSections.add(KV.second.toSpan<char>()); 600 else if (KV.first == "__DATA,__mod_init_func") 601 JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>()); 602 else { 603 // Should this be a warning instead? 604 return make_error<StringError>( 605 "Encountered unexpected section " + 606 std::string(KV.first.data(), KV.first.size()) + 607 " while registering object platform sections"); 608 } 609 } 610 611 return Error::success(); 612 } 613 614 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections( 615 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo, 616 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) { 617 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?) 618 // TODO: Add a JITDylib prepare-for-teardown operation that clears all 619 // registered sections, causing this function to take the fast-path. 620 ORC_RT_DEBUG({ 621 printdbg("MachOPlatform: Registering object sections for %p.\n", 622 HeaderAddr.toPtr<void *>()); 623 }); 624 625 std::lock_guard<std::mutex> Lock(JDStatesMutex); 626 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); 627 if (!JDS) { 628 std::ostringstream ErrStream; 629 ErrStream << "Could not register object platform sections for unrecognized " 630 "header " 631 << HeaderAddr.toPtr<void *>(); 632 return make_error<StringError>(ErrStream.str()); 633 } 634 635 // FIXME: Implement faster-path by returning immediately if JDS is being 636 // torn down entirely? 637 638 // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains 639 // any Swift or ObjC. Once this happens we can clear (and no longer record) 640 // data section content, as the library could never be re-initialized. 641 642 if (UnwindInfo && UseCallbackStyleUnwindInfo) { 643 ORC_RT_DEBUG({ 644 printdbg(" Deregistering new-style unwind info for:\n" 645 " DWARF: %p -- %p\n" 646 " Compact-unwind: %p -- %p\n" 647 " for:\n", 648 UnwindInfo->DwarfSection.Start.toPtr<void *>(), 649 UnwindInfo->DwarfSection.End.toPtr<void *>(), 650 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(), 651 UnwindInfo->CompactUnwindSection.End.toPtr<void *>()); 652 }); 653 for (auto &CodeRange : UnwindInfo->CodeRanges) { 654 JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(), 655 CodeRange.End.toPtr<char *>()); 656 ORC_RT_DEBUG({ 657 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(), 658 CodeRange.End.toPtr<void *>()); 659 }); 660 } 661 } 662 663 for (auto &KV : Secs) { 664 // FIXME: Validate section ranges? 665 if (KV.first == "__TEXT,__eh_frame") { 666 if (!UseCallbackStyleUnwindInfo) { 667 // Use classic libunwind registration. 668 if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>())) 669 return Err; 670 } 671 } else if (KV.first == "__DATA,__data") { 672 JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>()); 673 } else if (KV.first == "__DATA,__common") { 674 JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>()); 675 } else if (KV.first == "__DATA,__thread_data") { 676 if (auto Err = 677 deregisterThreadDataSection(KV.second.toSpan<const char>())) 678 return Err; 679 } else if (KV.first == "__DATA,__objc_selrefs") 680 JDS->ObjCSelRefsSections.removeIfPresent(KV.second); 681 else if (KV.first == "__DATA,__objc_classlist") 682 JDS->ObjCClassListSections.removeIfPresent(KV.second); 683 else if (KV.first == "__TEXT,__swift5_protos") 684 JDS->Swift5ProtocolsSections.removeIfPresent(KV.second); 685 else if (KV.first == "__TEXT,__swift5_proto") 686 JDS->Swift5ProtocolConformancesSections.removeIfPresent(KV.second); 687 else if (KV.first == "__TEXT,__swift5_types") 688 JDS->Swift5TypesSections.removeIfPresent(KV.second); 689 else if (KV.first == "__DATA,__mod_init_func") 690 JDS->ModInitsSections.removeIfPresent(KV.second); 691 else { 692 // Should this be a warning instead? 693 return make_error<StringError>( 694 "Encountered unexpected section " + 695 std::string(KV.first.data(), KV.first.size()) + 696 " while deregistering object platform sections"); 697 } 698 } 699 return Error::success(); 700 } 701 702 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } 703 704 void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) { 705 ORC_RT_DEBUG({ 706 std::string S(Path.data(), Path.size()); 707 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str()); 708 }); 709 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex); 710 if (auto H = dlopenImpl(Path, Mode)) 711 return *H; 712 else { 713 // FIXME: Make dlerror thread safe. 714 DLFcnError = toString(H.takeError()); 715 return nullptr; 716 } 717 } 718 719 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { 720 ORC_RT_DEBUG({ 721 auto *JDS = getJITDylibStateByHeader(DSOHandle); 722 std::string DylibName; 723 if (JDS) { 724 std::string S; 725 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); 726 } else 727 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, 728 "invalid handle"); 729 }); 730 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex); 731 if (auto Err = dlcloseImpl(DSOHandle)) { 732 // FIXME: Make dlerror thread safe. 733 DLFcnError = toString(std::move(Err)); 734 return -1; 735 } 736 return 0; 737 } 738 739 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, 740 std::string_view Symbol) { 741 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); 742 if (!Addr) { 743 DLFcnError = toString(Addr.takeError()); 744 return 0; 745 } 746 747 return Addr->toPtr<void *>(); 748 } 749 750 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, 751 void *DSOHandle) { 752 // FIXME: Handle out-of-memory errors, returning -1 if OOM. 753 std::lock_guard<std::mutex> Lock(JDStatesMutex); 754 auto *JDS = getJITDylibStateByHeader(DSOHandle); 755 if (!JDS) { 756 ORC_RT_DEBUG({ 757 printdbg("MachOPlatformRuntimeState::registerAtExit called with " 758 "unrecognized dso handle %p\n", 759 DSOHandle); 760 }); 761 return -1; 762 } 763 JDS->AtExits.push_back({F, Arg}); 764 return 0; 765 } 766 767 void MachOPlatformRuntimeState::runAtExits( 768 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { 769 auto AtExits = std::move(JDS.AtExits); 770 771 // Unlock while running atexits, as they may trigger operations that modify 772 // JDStates. 773 JDStatesLock.unlock(); 774 while (!AtExits.empty()) { 775 auto &AE = AtExits.back(); 776 AE.Func(AE.Arg); 777 AtExits.pop_back(); 778 } 779 JDStatesLock.lock(); 780 } 781 782 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { 783 std::unique_lock<std::mutex> Lock(JDStatesMutex); 784 auto *JDS = getJITDylibStateByHeader(DSOHandle); 785 ORC_RT_DEBUG({ 786 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized " 787 "dso_handle %p\n", 788 DSOHandle); 789 }); 790 if (JDS) 791 runAtExits(Lock, *JDS); 792 } 793 794 Expected<std::pair<const char *, size_t>> 795 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { 796 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 797 auto I = ThreadDataSections.upper_bound(ThreadData); 798 // Check that we have a valid entry covering this address. 799 if (I == ThreadDataSections.begin()) 800 return make_error<StringError>("No thread local data section for key"); 801 I = std::prev(I); 802 if (ThreadData >= I->first + I->second) 803 return make_error<StringError>("No thread local data section for key"); 804 return *I; 805 } 806 807 MachOPlatformRuntimeState::JITDylibState * 808 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) { 809 auto I = JDStates.find(DSOHandle); 810 if (I == JDStates.end()) { 811 I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first; 812 I->second.Header = DSOHandle; 813 } 814 return &I->second; 815 } 816 817 MachOPlatformRuntimeState::JITDylibState * 818 MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) { 819 // FIXME: Avoid creating string once we have C++20. 820 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); 821 if (I != JDNameToHeader.end()) 822 return getJITDylibStateByHeader(I->second); 823 return nullptr; 824 } 825 826 Expected<ExecutorAddr> 827 MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, 828 std::string_view Sym) { 829 Expected<ExecutorAddr> Result((ExecutorAddr())); 830 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( 831 SPSExecutorAddr, SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag, 832 Result, 833 ExecutorAddr::fromPtr(DSOHandle), 834 Sym)) 835 return std::move(Err); 836 return Result; 837 } 838 839 // eh-frame registration functions. 840 // We expect these to be available for all processes. 841 extern "C" void __register_frame(const void *); 842 extern "C" void __deregister_frame(const void *); 843 844 template <typename HandleFDEFn> 845 void walkEHFrameSection(span<const char> EHFrameSection, 846 HandleFDEFn HandleFDE) { 847 const char *CurCFIRecord = EHFrameSection.data(); 848 uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); 849 850 while (CurCFIRecord != EHFrameSection.end() && Size != 0) { 851 const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); 852 if (Size == 0xffffffff) 853 Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; 854 else 855 Size += 4; 856 uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); 857 858 if (Offset != 0) 859 HandleFDE(CurCFIRecord); 860 861 CurCFIRecord += Size; 862 Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); 863 } 864 } 865 866 bool MachOPlatformRuntimeState::lookupUnwindSections( 867 void *Addr, unw_dynamic_unwind_sections &Info) { 868 ORC_RT_DEBUG( 869 { printdbg("Tried to lookup unwind-info via new lookup call.\n"); }); 870 std::lock_guard<std::mutex> Lock(JDStatesMutex); 871 for (auto &KV : JDStates) { 872 auto &JD = KV.second; 873 auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr)); 874 if (I != JD.UnwindSections.end()) { 875 Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header); 876 Info.dwarf_section = 877 reinterpret_cast<uintptr_t>(I->second.DwarfSection.data()); 878 Info.dwarf_section_length = I->second.DwarfSection.size(); 879 Info.compact_unwind_section = 880 reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data()); 881 Info.compact_unwind_section_length = 882 I->second.CompactUnwindSection.size(); 883 return true; 884 } 885 } 886 return false; 887 } 888 889 int MachOPlatformRuntimeState::findDynamicUnwindSections( 890 uintptr_t addr, unw_dynamic_unwind_sections *info) { 891 if (!info) 892 return 0; 893 return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr, 894 *info); 895 } 896 897 Error MachOPlatformRuntimeState::registerEHFrames( 898 span<const char> EHFrameSection) { 899 walkEHFrameSection(EHFrameSection, __register_frame); 900 return Error::success(); 901 } 902 903 Error MachOPlatformRuntimeState::deregisterEHFrames( 904 span<const char> EHFrameSection) { 905 walkEHFrameSection(EHFrameSection, __deregister_frame); 906 return Error::success(); 907 } 908 909 Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) { 910 if (!JDS.ObjCSelRefsSections.hasNewSections()) 911 return Error::success(); 912 913 if (ORC_RT_UNLIKELY(!sel_registerName)) 914 return make_error<StringError>("sel_registerName is not available"); 915 916 JDS.ObjCSelRefsSections.processNewSections([](span<void *> SelRefs) { 917 for (void *&SelEntry : SelRefs) { 918 const char *SelName = reinterpret_cast<const char *>(SelEntry); 919 auto Sel = sel_registerName(SelName); 920 *reinterpret_cast<SEL *>(&SelEntry) = Sel; 921 } 922 }); 923 924 return Error::success(); 925 } 926 927 Error MachOPlatformRuntimeState::registerObjCClasses(JITDylibState &JDS) { 928 if (!JDS.ObjCClassListSections.hasNewSections()) 929 return Error::success(); 930 931 if (ORC_RT_UNLIKELY(!objc_msgSend)) 932 return make_error<StringError>("objc_msgSend is not available"); 933 if (ORC_RT_UNLIKELY(!objc_readClassPair)) 934 return make_error<StringError>("objc_readClassPair is not available"); 935 936 struct ObjCClassCompiled { 937 void *Metaclass; 938 void *Parent; 939 void *Cache1; 940 void *Cache2; 941 void *Data; 942 }; 943 944 auto ClassSelector = sel_registerName("class"); 945 946 return JDS.ObjCClassListSections.processNewSections( 947 [&](span<void *> ClassPtrs) -> Error { 948 for (void *ClassPtr : ClassPtrs) { 949 auto *Cls = reinterpret_cast<Class>(ClassPtr); 950 auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr); 951 objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), 952 ClassSelector); 953 auto Registered = objc_readClassPair(Cls, JDS.ObjCImageInfo); 954 // FIXME: Improve diagnostic by reporting the failed class's name. 955 if (Registered != Cls) 956 return make_error<StringError>( 957 "Unable to register Objective-C class"); 958 } 959 return Error::success(); 960 }); 961 } 962 963 Error MachOPlatformRuntimeState::registerSwift5Protocols(JITDylibState &JDS) { 964 965 if (!JDS.Swift5ProtocolsSections.hasNewSections()) 966 return Error::success(); 967 968 if (ORC_RT_UNLIKELY(!swift_registerProtocols)) 969 return make_error<StringError>("swift_registerProtocols is not available"); 970 971 JDS.Swift5ProtocolsSections.processNewSections([](span<char> ProtoSec) { 972 swift_registerProtocols( 973 reinterpret_cast<const ProtocolRecord *>(ProtoSec.data()), 974 reinterpret_cast<const ProtocolRecord *>(ProtoSec.data() + 975 ProtoSec.size())); 976 }); 977 978 return Error::success(); 979 } 980 981 Error MachOPlatformRuntimeState::registerSwift5ProtocolConformances( 982 JITDylibState &JDS) { 983 984 if (!JDS.Swift5ProtocolConformancesSections.hasNewSections()) 985 return Error::success(); 986 987 if (ORC_RT_UNLIKELY(!swift_registerProtocolConformances)) 988 return make_error<StringError>( 989 "swift_registerProtocolConformances is not available"); 990 991 JDS.Swift5ProtocolConformancesSections.processNewSections( 992 [](span<char> ProtoConfSec) { 993 swift_registerProtocolConformances( 994 reinterpret_cast<const ProtocolConformanceRecord *>( 995 ProtoConfSec.data()), 996 reinterpret_cast<const ProtocolConformanceRecord *>( 997 ProtoConfSec.data() + ProtoConfSec.size())); 998 }); 999 1000 return Error::success(); 1001 } 1002 1003 Error MachOPlatformRuntimeState::registerSwift5Types(JITDylibState &JDS) { 1004 1005 if (!JDS.Swift5TypesSections.hasNewSections()) 1006 return Error::success(); 1007 1008 if (ORC_RT_UNLIKELY(!swift_registerTypeMetadataRecords)) 1009 return make_error<StringError>( 1010 "swift_registerTypeMetadataRecords is not available"); 1011 1012 JDS.Swift5TypesSections.processNewSections([&](span<char> TypesSec) { 1013 swift_registerTypeMetadataRecords( 1014 reinterpret_cast<const TypeMetadataRecord *>(TypesSec.data()), 1015 reinterpret_cast<const TypeMetadataRecord *>(TypesSec.data() + 1016 TypesSec.size())); 1017 }); 1018 1019 return Error::success(); 1020 } 1021 1022 Error MachOPlatformRuntimeState::runModInits( 1023 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { 1024 std::vector<span<void (*)()>> InitSections; 1025 InitSections.reserve(JDS.ModInitsSections.numNewSections()); 1026 1027 // Copy initializer sections: If the JITDylib is unsealed then the 1028 // initializers could reach back into the JIT and cause more initializers to 1029 // be added. 1030 // FIXME: Skip unlock and run in-place on sealed JITDylibs? 1031 JDS.ModInitsSections.processNewSections( 1032 [&](span<void (*)()> Inits) { InitSections.push_back(Inits); }); 1033 1034 JDStatesLock.unlock(); 1035 for (auto InitSec : InitSections) 1036 for (auto *Init : InitSec) 1037 Init(); 1038 JDStatesLock.lock(); 1039 1040 return Error::success(); 1041 } 1042 1043 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path, 1044 int Mode) { 1045 std::unique_lock<std::mutex> Lock(JDStatesMutex); 1046 1047 // Try to find JITDylib state by name. 1048 auto *JDS = getJITDylibStateByName(Path); 1049 1050 if (!JDS) 1051 return make_error<StringError>("No registered JTIDylib for path " + 1052 std::string(Path.data(), Path.size())); 1053 1054 // If this JITDylib is unsealed, or this is the first dlopen then run 1055 // full dlopen path (update deps, push and run initializers, update ref 1056 // counts on all JITDylibs in the dep tree). 1057 if (!JDS->referenced() || !JDS->Sealed) { 1058 if (auto Err = dlopenFull(Lock, *JDS)) 1059 return std::move(Err); 1060 } 1061 1062 // Bump the ref-count on this dylib. 1063 ++JDS->DlRefCount; 1064 1065 // Return the header address. 1066 return JDS->Header; 1067 } 1068 1069 Error MachOPlatformRuntimeState::dlopenFull( 1070 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { 1071 // Call back to the JIT to push the initializers. 1072 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap())); 1073 // Unlock so that we can accept the initializer update. 1074 JDStatesLock.unlock(); 1075 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>( 1076 SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag, 1077 DepInfo, ExecutorAddr::fromPtr(JDS.Header))) 1078 return Err; 1079 JDStatesLock.lock(); 1080 1081 if (!DepInfo) 1082 return DepInfo.takeError(); 1083 1084 if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo)) 1085 return Err; 1086 1087 if (!DepInfo->empty()) { 1088 ORC_RT_DEBUG({ 1089 printdbg("Unrecognized dep-info key headers in dlopen of %s\n", 1090 JDS.Name.c_str()); 1091 }); 1092 std::ostringstream ErrStream; 1093 ErrStream << "Encountered unrecognized dep-info key headers " 1094 "while processing dlopen of " 1095 << JDS.Name; 1096 return make_error<StringError>(ErrStream.str()); 1097 } 1098 1099 return Error::success(); 1100 } 1101 1102 Error MachOPlatformRuntimeState::dlopenInitialize( 1103 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS, 1104 MachOJITDylibDepInfoMap &DepInfo) { 1105 ORC_RT_DEBUG({ 1106 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n", 1107 JDS.Name.c_str()); 1108 }); 1109 1110 // If the header is not present in the dep map then assume that we 1111 // already processed it earlier in the dlopenInitialize traversal and 1112 // return. 1113 // TODO: Keep a visited set instead so that we can error out on missing 1114 // entries? 1115 auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); 1116 if (I == DepInfo.end()) 1117 return Error::success(); 1118 1119 auto DI = std::move(I->second); 1120 DepInfo.erase(I); 1121 1122 // We don't need to re-initialize sealed JITDylibs that have already been 1123 // initialized. Just check that their dep-map entry is empty as expected. 1124 if (JDS.Sealed) { 1125 if (!DI.DepHeaders.empty()) { 1126 std::ostringstream ErrStream; 1127 ErrStream << "Sealed JITDylib " << JDS.Header 1128 << " already has registered dependencies"; 1129 return make_error<StringError>(ErrStream.str()); 1130 } 1131 if (JDS.referenced()) 1132 return Error::success(); 1133 } else 1134 JDS.Sealed = DI.Sealed; 1135 1136 // This is an unsealed or newly sealed JITDylib. Run initializers. 1137 std::vector<JITDylibState *> OldDeps; 1138 std::swap(JDS.Deps, OldDeps); 1139 JDS.Deps.reserve(DI.DepHeaders.size()); 1140 for (auto DepHeaderAddr : DI.DepHeaders) { 1141 auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>()); 1142 if (!DepJDS) { 1143 std::ostringstream ErrStream; 1144 ErrStream << "Encountered unrecognized dep header " 1145 << DepHeaderAddr.toPtr<void *>() << " while initializing " 1146 << JDS.Name; 1147 return make_error<StringError>(ErrStream.str()); 1148 } 1149 ++DepJDS->LinkedAgainstRefCount; 1150 if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo)) 1151 return Err; 1152 } 1153 1154 // Initialize this JITDylib. 1155 if (auto Err = registerObjCSelectors(JDS)) 1156 return Err; 1157 if (auto Err = registerObjCClasses(JDS)) 1158 return Err; 1159 if (auto Err = registerSwift5Protocols(JDS)) 1160 return Err; 1161 if (auto Err = registerSwift5ProtocolConformances(JDS)) 1162 return Err; 1163 if (auto Err = registerSwift5Types(JDS)) 1164 return Err; 1165 if (auto Err = runModInits(JDStatesLock, JDS)) 1166 return Err; 1167 1168 // Decrement old deps. 1169 // FIXME: We should probably continue and just report deinitialize errors 1170 // here. 1171 for (auto *DepJDS : OldDeps) { 1172 --DepJDS->LinkedAgainstRefCount; 1173 if (!DepJDS->referenced()) 1174 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS)) 1175 return Err; 1176 } 1177 1178 return Error::success(); 1179 } 1180 1181 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { 1182 std::unique_lock<std::mutex> Lock(JDStatesMutex); 1183 1184 // Try to find JITDylib state by header. 1185 auto *JDS = getJITDylibStateByHeader(DSOHandle); 1186 1187 if (!JDS) { 1188 std::ostringstream ErrStream; 1189 ErrStream << "No registered JITDylib for " << DSOHandle; 1190 return make_error<StringError>(ErrStream.str()); 1191 } 1192 1193 // Bump the ref-count. 1194 --JDS->DlRefCount; 1195 1196 if (!JDS->referenced()) 1197 return dlcloseDeinitialize(Lock, *JDS); 1198 1199 return Error::success(); 1200 } 1201 1202 Error MachOPlatformRuntimeState::dlcloseDeinitialize( 1203 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) { 1204 1205 ORC_RT_DEBUG({ 1206 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", 1207 JDS.Name.c_str()); 1208 }); 1209 1210 runAtExits(JDStatesLock, JDS); 1211 1212 // Reset mod-inits 1213 JDS.ModInitsSections.reset(); 1214 1215 // Reset data section contents. 1216 for (auto &KV : JDS.DataSectionContent) 1217 memcpy(KV.first, KV.second.data(), KV.second.size()); 1218 for (auto &KV : JDS.ZeroInitRanges) 1219 memset(KV.first, 0, KV.second); 1220 1221 // Deinitialize any dependencies. 1222 for (auto *DepJDS : JDS.Deps) { 1223 --DepJDS->LinkedAgainstRefCount; 1224 if (!DepJDS->referenced()) 1225 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS)) 1226 return Err; 1227 } 1228 1229 return Error::success(); 1230 } 1231 1232 class MachOPlatformRuntimeTLVManager { 1233 public: 1234 void *getInstance(const char *ThreadData); 1235 1236 private: 1237 std::unordered_map<const char *, char *> Instances; 1238 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; 1239 }; 1240 1241 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { 1242 auto I = Instances.find(ThreadData); 1243 if (I != Instances.end()) 1244 return I->second; 1245 1246 auto TDS = 1247 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); 1248 if (!TDS) { 1249 __orc_rt_log_error(toString(TDS.takeError()).c_str()); 1250 return nullptr; 1251 } 1252 1253 auto &Allocated = AllocatedSections[TDS->first]; 1254 if (!Allocated) { 1255 Allocated = std::make_unique<char[]>(TDS->second); 1256 memcpy(Allocated.get(), TDS->first, TDS->second); 1257 } 1258 1259 size_t ThreadDataDelta = ThreadData - TDS->first; 1260 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); 1261 1262 char *Instance = Allocated.get() + ThreadDataDelta; 1263 Instances[ThreadData] = Instance; 1264 return Instance; 1265 } 1266 1267 void destroyMachOTLVMgr(void *MachOTLVMgr) { 1268 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr); 1269 } 1270 1271 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) { 1272 for (auto &WFC : WFCs) 1273 if (auto Err = WFC.runWithSPSRet<void>()) 1274 return Err; 1275 return Error::success(); 1276 } 1277 1278 } // end anonymous namespace 1279 1280 //------------------------------------------------------------------------------ 1281 // JIT entry points 1282 //------------------------------------------------------------------------------ 1283 1284 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1285 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) { 1286 return WrapperFunction<SPSError()>::handle( 1287 ArgData, ArgSize, 1288 []() { return MachOPlatformRuntimeState::create(); }) 1289 .release(); 1290 } 1291 1292 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1293 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { 1294 return WrapperFunction<SPSError()>::handle( 1295 ArgData, ArgSize, 1296 []() { return MachOPlatformRuntimeState::destroy(); }) 1297 .release(); 1298 } 1299 1300 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1301 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) { 1302 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle( 1303 ArgData, ArgSize, 1304 [](std::string &Name, ExecutorAddr HeaderAddr) { 1305 return MachOPlatformRuntimeState::get().registerJITDylib( 1306 std::move(Name), HeaderAddr.toPtr<void *>()); 1307 }) 1308 .release(); 1309 } 1310 1311 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1312 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) { 1313 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( 1314 ArgData, ArgSize, 1315 [](ExecutorAddr HeaderAddr) { 1316 return MachOPlatformRuntimeState::get().deregisterJITDylib( 1317 HeaderAddr.toPtr<void *>()); 1318 }) 1319 .release(); 1320 } 1321 1322 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1323 __orc_rt_macho_register_object_platform_sections(char *ArgData, 1324 size_t ArgSize) { 1325 return WrapperFunction<SPSError(SPSExecutorAddr, 1326 SPSOptional<SPSUnwindSectionInfo>, 1327 SPSMachOObjectPlatformSectionsMap)>:: 1328 handle(ArgData, ArgSize, 1329 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI, 1330 std::vector<std::pair<std::string_view, ExecutorAddrRange>> 1331 &Secs) { 1332 return MachOPlatformRuntimeState::get() 1333 .registerObjectPlatformSections(HeaderAddr, std::move(USI), 1334 std::move(Secs)); 1335 }) 1336 .release(); 1337 } 1338 1339 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1340 __orc_rt_macho_deregister_object_platform_sections(char *ArgData, 1341 size_t ArgSize) { 1342 return WrapperFunction<SPSError(SPSExecutorAddr, 1343 SPSOptional<SPSUnwindSectionInfo>, 1344 SPSMachOObjectPlatformSectionsMap)>:: 1345 handle(ArgData, ArgSize, 1346 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI, 1347 std::vector<std::pair<std::string_view, ExecutorAddrRange>> 1348 &Secs) { 1349 return MachOPlatformRuntimeState::get() 1350 .deregisterObjectPlatformSections(HeaderAddr, std::move(USI), 1351 std::move(Secs)); 1352 }) 1353 .release(); 1354 } 1355 1356 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1357 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) { 1358 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle( 1359 ArgData, ArgSize, runWrapperFunctionCalls) 1360 .release(); 1361 } 1362 1363 //------------------------------------------------------------------------------ 1364 // TLV support 1365 //------------------------------------------------------------------------------ 1366 1367 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) { 1368 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>( 1369 pthread_getspecific(D->Key)); 1370 if (!TLVMgr) { 1371 TLVMgr = new MachOPlatformRuntimeTLVManager(); 1372 if (pthread_setspecific(D->Key, TLVMgr)) { 1373 __orc_rt_log_error("Call to pthread_setspecific failed"); 1374 return nullptr; 1375 } 1376 } 1377 1378 return TLVMgr->getInstance( 1379 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); 1380 } 1381 1382 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1383 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) { 1384 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( 1385 ArgData, ArgSize, 1386 []() -> Expected<uint64_t> { 1387 pthread_key_t Key; 1388 if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) { 1389 __orc_rt_log_error("Call to pthread_key_create failed"); 1390 return make_error<StringError>(strerror(Err)); 1391 } 1392 return static_cast<uint64_t>(Key); 1393 }) 1394 .release(); 1395 } 1396 1397 //------------------------------------------------------------------------------ 1398 // cxa_atexit support 1399 //------------------------------------------------------------------------------ 1400 1401 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, 1402 void *dso_handle) { 1403 return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle); 1404 } 1405 1406 void __orc_rt_macho_cxa_finalize(void *dso_handle) { 1407 MachOPlatformRuntimeState::get().runAtExits(dso_handle); 1408 } 1409 1410 //------------------------------------------------------------------------------ 1411 // JIT'd dlfcn alternatives. 1412 //------------------------------------------------------------------------------ 1413 1414 const char *__orc_rt_macho_jit_dlerror() { 1415 return MachOPlatformRuntimeState::get().dlerror(); 1416 } 1417 1418 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) { 1419 return MachOPlatformRuntimeState::get().dlopen(path, mode); 1420 } 1421 1422 int __orc_rt_macho_jit_dlclose(void *dso_handle) { 1423 return MachOPlatformRuntimeState::get().dlclose(dso_handle); 1424 } 1425 1426 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) { 1427 return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol); 1428 } 1429 1430 //------------------------------------------------------------------------------ 1431 // MachO Run Program 1432 //------------------------------------------------------------------------------ 1433 1434 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName, 1435 const char *EntrySymbolName, 1436 int argc, char *argv[]) { 1437 using MainTy = int (*)(int, char *[]); 1438 1439 void *H = __orc_rt_macho_jit_dlopen(JITDylibName, 1440 __orc_rt::macho::ORC_RT_RTLD_LAZY); 1441 if (!H) { 1442 __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); 1443 return -1; 1444 } 1445 1446 auto *Main = 1447 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName)); 1448 1449 if (!Main) { 1450 __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); 1451 return -1; 1452 } 1453 1454 int Result = Main(argc, argv); 1455 1456 if (__orc_rt_macho_jit_dlclose(H) == -1) 1457 __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); 1458 1459 return Result; 1460 } 1461