1 //===- macho_platform.cpp -------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains code required to load the rest of the MachO runtime. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "macho_platform.h" 14 #include "common.h" 15 #include "debug.h" 16 #include "error.h" 17 #include "wrapper_function_utils.h" 18 19 #include <algorithm> 20 #include <ios> 21 #include <map> 22 #include <mutex> 23 #include <sstream> 24 #include <unordered_map> 25 #include <unordered_set> 26 #include <vector> 27 28 #define DEBUG_TYPE "macho_platform" 29 30 using namespace __orc_rt; 31 using namespace __orc_rt::macho; 32 33 // Declare function tags for functions in the JIT process. 34 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag) 35 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_symbol_lookup_tag) 36 37 // Objective-C types. 38 struct objc_class; 39 struct objc_image_info; 40 struct objc_object; 41 struct objc_selector; 42 43 using Class = objc_class *; 44 using id = objc_object *; 45 using SEL = objc_selector *; 46 47 // Objective-C registration functions. 48 // These are weakly imported. If the Objective-C runtime has not been loaded 49 // then code containing Objective-C sections will generate an error. 50 extern "C" id objc_msgSend(id, SEL, ...) ORC_RT_WEAK_IMPORT; 51 extern "C" Class objc_readClassPair(Class, 52 const objc_image_info *) ORC_RT_WEAK_IMPORT; 53 extern "C" SEL sel_registerName(const char *) ORC_RT_WEAK_IMPORT; 54 55 // Swift types. 56 class ProtocolRecord; 57 class ProtocolConformanceRecord; 58 class TypeMetadataRecord; 59 60 extern "C" void 61 swift_registerProtocols(const ProtocolRecord *begin, 62 const ProtocolRecord *end) ORC_RT_WEAK_IMPORT; 63 64 extern "C" void swift_registerProtocolConformances( 65 const ProtocolConformanceRecord *begin, 66 const ProtocolConformanceRecord *end) ORC_RT_WEAK_IMPORT; 67 68 extern "C" void swift_registerTypeMetadataRecords( 69 const TypeMetadataRecord *begin, 70 const TypeMetadataRecord *end) ORC_RT_WEAK_IMPORT; 71 72 namespace { 73 74 struct MachOJITDylibDepInfo { 75 bool Sealed = false; 76 std::vector<ExecutorAddr> DepHeaders; 77 }; 78 79 using MachOJITDylibDepInfoMap = 80 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>; 81 82 } // anonymous namespace 83 84 namespace __orc_rt { 85 86 using SPSMachOObjectPlatformSectionsMap = 87 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; 88 89 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>; 90 91 using SPSMachOJITDylibDepInfoMap = 92 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>; 93 94 template <> 95 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> { 96 public: 97 static size_t size(const MachOJITDylibDepInfo &JDI) { 98 return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders); 99 } 100 101 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) { 102 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed, 103 JDI.DepHeaders); 104 } 105 106 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) { 107 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed, 108 JDI.DepHeaders); 109 } 110 }; 111 112 } // namespace __orc_rt 113 114 namespace { 115 struct TLVDescriptor { 116 void *(*Thunk)(TLVDescriptor *) = nullptr; 117 unsigned long Key = 0; 118 unsigned long DataAddress = 0; 119 }; 120 121 class MachOPlatformRuntimeState { 122 private: 123 struct AtExitEntry { 124 void (*Func)(void *); 125 void *Arg; 126 }; 127 128 using AtExitsVector = std::vector<AtExitEntry>; 129 130 struct JITDylibState { 131 std::string Name; 132 void *Header = nullptr; 133 bool Sealed = false; 134 size_t LinkedAgainstRefCount = 0; 135 size_t DlRefCount = 0; 136 std::vector<JITDylibState *> Deps; 137 AtExitsVector AtExits; 138 const objc_image_info *ObjCImageInfo = nullptr; 139 std::vector<span<void (*)()>> ModInitsSections; 140 std::vector<span<void (*)()>> ModInitsSectionsNew; 141 std::vector<span<uintptr_t>> ObjCClassListSections; 142 std::vector<span<uintptr_t>> ObjCClassListSectionsNew; 143 std::vector<span<uintptr_t>> ObjCSelRefsSections; 144 std::vector<span<uintptr_t>> ObjCSelRefsSectionsNew; 145 std::vector<span<char>> Swift5ProtoSections; 146 std::vector<span<char>> Swift5ProtoSectionsNew; 147 std::vector<span<char>> Swift5ProtosSections; 148 std::vector<span<char>> Swift5ProtosSectionsNew; 149 std::vector<span<char>> Swift5TypesSections; 150 std::vector<span<char>> Swift5TypesSectionsNew; 151 152 bool referenced() const { 153 return LinkedAgainstRefCount != 0 || DlRefCount != 0; 154 } 155 }; 156 157 public: 158 static void initialize(); 159 static MachOPlatformRuntimeState &get(); 160 static void destroy(); 161 162 MachOPlatformRuntimeState() = default; 163 164 // Delete copy and move constructors. 165 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete; 166 MachOPlatformRuntimeState & 167 operator=(const MachOPlatformRuntimeState &) = delete; 168 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete; 169 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete; 170 171 Error registerJITDylib(std::string Name, void *Header); 172 Error deregisterJITDylib(void *Header); 173 Error registerThreadDataSection(span<const char> ThreadDataSection); 174 Error deregisterThreadDataSection(span<const char> ThreadDataSection); 175 Error registerObjectPlatformSections( 176 ExecutorAddr HeaderAddr, 177 std::vector<std::pair<string_view, ExecutorAddrRange>> Secs); 178 Error deregisterObjectPlatformSections( 179 ExecutorAddr HeaderAddr, 180 std::vector<std::pair<string_view, ExecutorAddrRange>> Secs); 181 182 const char *dlerror(); 183 void *dlopen(string_view Name, int Mode); 184 int dlclose(void *DSOHandle); 185 void *dlsym(void *DSOHandle, string_view Symbol); 186 187 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle); 188 void runAtExits(JITDylibState &JDS); 189 void runAtExits(void *DSOHandle); 190 191 /// Returns the base address of the section containing ThreadData. 192 Expected<std::pair<const char *, size_t>> 193 getThreadDataSectionFor(const char *ThreadData); 194 195 private: 196 JITDylibState *getJITDylibStateByHeader(void *DSOHandle); 197 JITDylibState *getJITDylibStateByName(string_view Path); 198 199 Expected<ExecutorAddr> lookupSymbolInJITDylib(void *DSOHandle, 200 string_view Symbol); 201 202 static Error registerObjCSelectors(JITDylibState &JDS); 203 static Error registerObjCClasses(JITDylibState &JDS); 204 static Error registerSwift5Protocols(JITDylibState &JDS); 205 static Error registerSwift5ProtocolConformances(JITDylibState &JDS); 206 static Error registerSwift5Types(JITDylibState &JDS); 207 static Error runModInits(JITDylibState &JDS); 208 209 Expected<void *> dlopenImpl(string_view Path, int Mode); 210 Error dlopenFull(JITDylibState &JDS); 211 Error dlopenInitialize(JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo); 212 213 Error dlcloseImpl(void *DSOHandle); 214 Error dlcloseDeinitialize(JITDylibState &JDS); 215 216 static MachOPlatformRuntimeState *MOPS; 217 218 // FIXME: Move to thread-state. 219 std::string DLFcnError; 220 221 std::recursive_mutex JDStatesMutex; 222 std::unordered_map<void *, JITDylibState> JDStates; 223 std::unordered_map<string_view, void *> JDNameToHeader; 224 225 std::mutex ThreadDataSectionsMutex; 226 std::map<const char *, size_t> ThreadDataSections; 227 }; 228 229 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr; 230 231 void MachOPlatformRuntimeState::initialize() { 232 assert(!MOPS && "MachOPlatformRuntimeState should be null"); 233 MOPS = new MachOPlatformRuntimeState(); 234 } 235 236 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() { 237 assert(MOPS && "MachOPlatformRuntimeState not initialized"); 238 return *MOPS; 239 } 240 241 void MachOPlatformRuntimeState::destroy() { 242 assert(MOPS && "MachOPlatformRuntimeState not initialized"); 243 delete MOPS; 244 } 245 246 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name, 247 void *Header) { 248 ORC_RT_DEBUG({ 249 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header); 250 }); 251 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 252 if (JDStates.count(Header)) { 253 std::ostringstream ErrStream; 254 ErrStream << "Duplicate JITDylib registration for header " << Header 255 << " (name = " << Name << ")"; 256 return make_error<StringError>(ErrStream.str()); 257 } 258 if (JDNameToHeader.count(Name)) { 259 std::ostringstream ErrStream; 260 ErrStream << "Duplicate JITDylib registration for header " << Header 261 << " (header = " << Header << ")"; 262 return make_error<StringError>(ErrStream.str()); 263 } 264 265 auto &JDS = JDStates[Header]; 266 JDS.Name = std::move(Name); 267 JDS.Header = Header; 268 JDNameToHeader[JDS.Name] = Header; 269 return Error::success(); 270 } 271 272 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) { 273 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 274 auto I = JDStates.find(Header); 275 if (I == JDStates.end()) { 276 std::ostringstream ErrStream; 277 ErrStream << "Attempted to deregister unrecognized header " << Header; 278 return make_error<StringError>(ErrStream.str()); 279 } 280 281 // Remove std::string construction once we can use C++20. 282 auto J = JDNameToHeader.find( 283 std::string(I->second.Name.data(), I->second.Name.size())); 284 assert(J != JDNameToHeader.end() && 285 "Missing JDNameToHeader entry for JITDylib"); 286 287 ORC_RT_DEBUG({ 288 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(), 289 Header); 290 }); 291 292 JDNameToHeader.erase(J); 293 JDStates.erase(I); 294 return Error::success(); 295 } 296 297 Error MachOPlatformRuntimeState::registerThreadDataSection( 298 span<const char> ThreadDataSection) { 299 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 300 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data()); 301 if (I != ThreadDataSections.begin()) { 302 auto J = std::prev(I); 303 if (J->first + J->second > ThreadDataSection.data()) 304 return make_error<StringError>("Overlapping __thread_data sections"); 305 } 306 ThreadDataSections.insert( 307 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size())); 308 return Error::success(); 309 } 310 311 Error MachOPlatformRuntimeState::deregisterThreadDataSection( 312 span<const char> ThreadDataSection) { 313 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 314 auto I = ThreadDataSections.find(ThreadDataSection.data()); 315 if (I == ThreadDataSections.end()) 316 return make_error<StringError>("Attempt to deregister unknown thread data " 317 "section"); 318 ThreadDataSections.erase(I); 319 return Error::success(); 320 } 321 322 Error MachOPlatformRuntimeState::registerObjectPlatformSections( 323 ExecutorAddr HeaderAddr, 324 std::vector<std::pair<string_view, ExecutorAddrRange>> Secs) { 325 ORC_RT_DEBUG({ 326 printdbg("MachOPlatform: Registering object sections for %p.\n", 327 HeaderAddr.toPtr<void *>()); 328 }); 329 330 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 331 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); 332 if (!JDS) { 333 std::ostringstream ErrStream; 334 ErrStream << "Could not register object platform sections for " 335 "unrecognized header " 336 << HeaderAddr.toPtr<void *>(); 337 return make_error<StringError>(ErrStream.str()); 338 } 339 340 for (auto &KV : Secs) { 341 // FIXME: Validate section ranges? 342 if (KV.first == "__DATA,__thread_data") { 343 if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>())) 344 return Err; 345 } else if (KV.first == "__DATA,__objc_selrefs") 346 JDS->ObjCSelRefsSectionsNew.push_back(KV.second.toSpan<uintptr_t>()); 347 else if (KV.first == "__DATA,__objc_classlist") 348 JDS->ObjCClassListSectionsNew.push_back(KV.second.toSpan<uintptr_t>()); 349 else if (KV.first == "__TEXT,__swift5_protos") 350 JDS->Swift5ProtosSectionsNew.push_back(KV.second.toSpan<char>()); 351 else if (KV.first == "__TEXT,__swift5_proto") 352 JDS->Swift5ProtoSectionsNew.push_back(KV.second.toSpan<char>()); 353 else if (KV.first == "__TEXT,__swift5_types") 354 JDS->Swift5TypesSectionsNew.push_back(KV.second.toSpan<char>()); 355 else if (KV.first == "__DATA,__mod_init_func") 356 JDS->ModInitsSectionsNew.push_back(KV.second.toSpan<void (*)()>()); 357 else { 358 // Should this be a warning instead? 359 return make_error<StringError>( 360 "Encountered unexpected section " + 361 std::string(KV.first.data(), KV.first.size()) + 362 " while registering object platform sections"); 363 } 364 } 365 366 return Error::success(); 367 } 368 369 // Remove the given range from the given vector if present. 370 // Returns true if the range was removed, false otherwise. 371 template <typename T> 372 bool removeIfPresent(std::vector<span<T>> &V, ExecutorAddrRange R) { 373 auto RI = std::find_if( 374 V.rbegin(), V.rend(), 375 [RS = R.toSpan<T>()](const span<T> &E) { return E.data() == RS.data(); }); 376 if (RI != V.rend()) { 377 V.erase(std::next(RI).base()); 378 return true; 379 } 380 return false; 381 } 382 383 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections( 384 ExecutorAddr HeaderAddr, 385 std::vector<std::pair<string_view, ExecutorAddrRange>> Secs) { 386 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?) 387 // TODO: Add a JITDylib prepare-for-teardown operation that clears all 388 // registered sections, causing this function to take the fast-path. 389 ORC_RT_DEBUG({ 390 printdbg("MachOPlatform: Registering object sections for %p.\n", 391 HeaderAddr.toPtr<void *>()); 392 }); 393 394 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 395 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>()); 396 if (!JDS) { 397 std::ostringstream ErrStream; 398 ErrStream << "Could not register object platform sections for unrecognized " 399 "header " 400 << HeaderAddr.toPtr<void *>(); 401 return make_error<StringError>(ErrStream.str()); 402 } 403 404 // FIXME: Implement faster-path by returning immediately if JDS is being 405 // torn down entirely? 406 407 for (auto &KV : Secs) { 408 // FIXME: Validate section ranges? 409 if (KV.first == "__DATA,__thread_data") { 410 if (auto Err = 411 deregisterThreadDataSection(KV.second.toSpan<const char>())) 412 return Err; 413 } else if (KV.first == "__DATA,__objc_selrefs") { 414 if (!removeIfPresent(JDS->ObjCSelRefsSections, KV.second)) 415 removeIfPresent(JDS->ObjCSelRefsSectionsNew, KV.second); 416 } else if (KV.first == "__DATA,__objc_classlist") { 417 if (!removeIfPresent(JDS->ObjCClassListSections, KV.second)) 418 removeIfPresent(JDS->ObjCClassListSectionsNew, KV.second); 419 } else if (KV.first == "__TEXT,__swift5_protos") { 420 if (!removeIfPresent(JDS->Swift5ProtosSections, KV.second)) 421 removeIfPresent(JDS->Swift5ProtosSectionsNew, KV.second); 422 } else if (KV.first == "__TEXT,__swift5_proto") { 423 if (!removeIfPresent(JDS->Swift5ProtoSections, KV.second)) 424 removeIfPresent(JDS->Swift5ProtoSectionsNew, KV.second); 425 } else if (KV.first == "__TEXT,__swift5_types") { 426 if (!removeIfPresent(JDS->Swift5TypesSections, KV.second)) 427 removeIfPresent(JDS->Swift5TypesSectionsNew, KV.second); 428 } else if (KV.first == "__DATA,__mod_init_func") { 429 if (!removeIfPresent(JDS->ModInitsSections, KV.second)) 430 removeIfPresent(JDS->ModInitsSectionsNew, KV.second); 431 } else { 432 // Should this be a warning instead? 433 return make_error<StringError>( 434 "Encountered unexpected section " + 435 std::string(KV.first.data(), KV.first.size()) + 436 " while deregistering object platform sections"); 437 } 438 } 439 return Error::success(); 440 } 441 442 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); } 443 444 void *MachOPlatformRuntimeState::dlopen(string_view Path, int Mode) { 445 ORC_RT_DEBUG({ 446 std::string S(Path.data(), Path.size()); 447 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str()); 448 }); 449 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 450 if (auto H = dlopenImpl(Path, Mode)) 451 return *H; 452 else { 453 // FIXME: Make dlerror thread safe. 454 DLFcnError = toString(H.takeError()); 455 return nullptr; 456 } 457 } 458 459 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { 460 ORC_RT_DEBUG({ 461 auto *JDS = getJITDylibStateByHeader(DSOHandle); 462 std::string DylibName; 463 if (JDS) { 464 std::string S; 465 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str()); 466 } else 467 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, 468 "invalid handle"); 469 }); 470 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 471 if (auto Err = dlcloseImpl(DSOHandle)) { 472 // FIXME: Make dlerror thread safe. 473 DLFcnError = toString(std::move(Err)); 474 return -1; 475 } 476 return 0; 477 } 478 479 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, string_view Symbol) { 480 auto Addr = lookupSymbolInJITDylib(DSOHandle, Symbol); 481 if (!Addr) { 482 DLFcnError = toString(Addr.takeError()); 483 return 0; 484 } 485 486 return Addr->toPtr<void *>(); 487 } 488 489 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, 490 void *DSOHandle) { 491 // FIXME: Handle out-of-memory errors, returning -1 if OOM. 492 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 493 auto *JDS = getJITDylibStateByHeader(DSOHandle); 494 if (!JDS) { 495 ORC_RT_DEBUG({ 496 printdbg("MachOPlatformRuntimeState::registerAtExit called with " 497 "unrecognized dso handle %p\n", 498 DSOHandle); 499 }); 500 return -1; 501 } 502 JDS->AtExits.push_back({F, Arg}); 503 return 0; 504 } 505 506 void MachOPlatformRuntimeState::runAtExits(JITDylibState &JDS) { 507 while (!JDS.AtExits.empty()) { 508 auto &AE = JDS.AtExits.back(); 509 AE.Func(AE.Arg); 510 JDS.AtExits.pop_back(); 511 } 512 } 513 514 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) { 515 std::lock_guard<std::recursive_mutex> Lock(JDStatesMutex); 516 auto *JDS = getJITDylibStateByHeader(DSOHandle); 517 ORC_RT_DEBUG({ 518 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized " 519 "dso_handle %p\n", 520 DSOHandle); 521 }); 522 if (JDS) 523 runAtExits(*JDS); 524 } 525 526 Expected<std::pair<const char *, size_t>> 527 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) { 528 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex); 529 auto I = ThreadDataSections.upper_bound(ThreadData); 530 // Check that we have a valid entry covering this address. 531 if (I == ThreadDataSections.begin()) 532 return make_error<StringError>("No thread local data section for key"); 533 I = std::prev(I); 534 if (ThreadData >= I->first + I->second) 535 return make_error<StringError>("No thread local data section for key"); 536 return *I; 537 } 538 539 MachOPlatformRuntimeState::JITDylibState * 540 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) { 541 auto I = JDStates.find(DSOHandle); 542 if (I == JDStates.end()) { 543 I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first; 544 I->second.Header = DSOHandle; 545 } 546 return &I->second; 547 } 548 549 MachOPlatformRuntimeState::JITDylibState * 550 MachOPlatformRuntimeState::getJITDylibStateByName(string_view Name) { 551 // FIXME: Avoid creating string once we have C++20. 552 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size())); 553 if (I != JDNameToHeader.end()) 554 return getJITDylibStateByHeader(I->second); 555 return nullptr; 556 } 557 558 Expected<ExecutorAddr> 559 MachOPlatformRuntimeState::lookupSymbolInJITDylib(void *DSOHandle, 560 string_view Sym) { 561 Expected<ExecutorAddr> Result((ExecutorAddr())); 562 if (auto Err = WrapperFunction<SPSExpected<SPSExecutorAddr>( 563 SPSExecutorAddr, SPSString)>::call(&__orc_rt_macho_symbol_lookup_tag, 564 Result, 565 ExecutorAddr::fromPtr(DSOHandle), 566 Sym)) 567 return std::move(Err); 568 return Result; 569 } 570 571 template <typename T> 572 static void moveAppendSections(std::vector<span<T>> &Dst, 573 std::vector<span<T>> &Src) { 574 if (Dst.empty()) { 575 Dst = std::move(Src); 576 return; 577 } 578 579 Dst.reserve(Dst.size() + Src.size()); 580 std::copy(Src.begin(), Src.end(), std::back_inserter(Dst)); 581 Src.clear(); 582 } 583 584 Error MachOPlatformRuntimeState::registerObjCSelectors(JITDylibState &JDS) { 585 586 if (JDS.ObjCSelRefsSectionsNew.empty()) 587 return Error::success(); 588 589 if (ORC_RT_UNLIKELY(!sel_registerName)) 590 return make_error<StringError>("sel_registerName is not available"); 591 592 for (const auto &ObjCSelRefs : JDS.ObjCSelRefsSectionsNew) { 593 for (uintptr_t &SelEntry : ObjCSelRefs) { 594 const char *SelName = reinterpret_cast<const char *>(SelEntry); 595 auto Sel = sel_registerName(SelName); 596 *reinterpret_cast<SEL *>(&SelEntry) = Sel; 597 } 598 } 599 600 moveAppendSections(JDS.ObjCSelRefsSections, JDS.ObjCSelRefsSectionsNew); 601 return Error::success(); 602 } 603 604 Error MachOPlatformRuntimeState::registerObjCClasses(JITDylibState &JDS) { 605 606 if (JDS.ObjCClassListSectionsNew.empty()) 607 return Error::success(); 608 609 if (ORC_RT_UNLIKELY(!objc_msgSend)) 610 return make_error<StringError>("objc_msgSend is not available"); 611 if (ORC_RT_UNLIKELY(!objc_readClassPair)) 612 return make_error<StringError>("objc_readClassPair is not available"); 613 614 struct ObjCClassCompiled { 615 void *Metaclass; 616 void *Parent; 617 void *Cache1; 618 void *Cache2; 619 void *Data; 620 }; 621 622 auto ClassSelector = sel_registerName("class"); 623 624 for (const auto &ObjCClassList : JDS.ObjCClassListSectionsNew) { 625 for (uintptr_t ClassPtr : ObjCClassList) { 626 auto *Cls = reinterpret_cast<Class>(ClassPtr); 627 auto *ClassCompiled = reinterpret_cast<ObjCClassCompiled *>(ClassPtr); 628 objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector); 629 auto Registered = objc_readClassPair(Cls, JDS.ObjCImageInfo); 630 631 // FIXME: Improve diagnostic by reporting the failed class's name. 632 if (Registered != Cls) 633 return make_error<StringError>("Unable to register Objective-C class"); 634 } 635 } 636 637 moveAppendSections(JDS.ObjCClassListSections, JDS.ObjCClassListSectionsNew); 638 return Error::success(); 639 } 640 641 Error MachOPlatformRuntimeState::registerSwift5Protocols(JITDylibState &JDS) { 642 643 if (JDS.Swift5ProtosSectionsNew.empty()) 644 return Error::success(); 645 646 if (ORC_RT_UNLIKELY(!swift_registerProtocols)) 647 return make_error<StringError>("swift_registerProtocols is not available"); 648 649 for (const auto &Swift5Protocols : JDS.Swift5ProtoSectionsNew) 650 swift_registerProtocols( 651 reinterpret_cast<const ProtocolRecord *>(Swift5Protocols.data()), 652 reinterpret_cast<const ProtocolRecord *>(Swift5Protocols.data() + 653 Swift5Protocols.size())); 654 655 moveAppendSections(JDS.Swift5ProtoSections, JDS.Swift5ProtoSectionsNew); 656 return Error::success(); 657 } 658 659 Error MachOPlatformRuntimeState::registerSwift5ProtocolConformances( 660 JITDylibState &JDS) { 661 662 if (JDS.Swift5ProtosSectionsNew.empty()) 663 return Error::success(); 664 665 if (ORC_RT_UNLIKELY(!swift_registerProtocolConformances)) 666 return make_error<StringError>( 667 "swift_registerProtocolConformances is not available"); 668 669 for (const auto &ProtoConfSec : JDS.Swift5ProtosSectionsNew) 670 swift_registerProtocolConformances( 671 reinterpret_cast<const ProtocolConformanceRecord *>( 672 ProtoConfSec.data()), 673 reinterpret_cast<const ProtocolConformanceRecord *>( 674 ProtoConfSec.data() + ProtoConfSec.size())); 675 676 moveAppendSections(JDS.Swift5ProtosSections, JDS.Swift5ProtosSectionsNew); 677 return Error::success(); 678 } 679 680 Error MachOPlatformRuntimeState::registerSwift5Types(JITDylibState &JDS) { 681 682 if (JDS.Swift5TypesSectionsNew.empty()) 683 return Error::success(); 684 685 if (ORC_RT_UNLIKELY(!swift_registerTypeMetadataRecords)) 686 return make_error<StringError>( 687 "swift_registerTypeMetadataRecords is not available"); 688 689 for (const auto &TypeSec : JDS.Swift5TypesSectionsNew) 690 swift_registerTypeMetadataRecords( 691 reinterpret_cast<const TypeMetadataRecord *>(TypeSec.data()), 692 reinterpret_cast<const TypeMetadataRecord *>(TypeSec.data() + 693 TypeSec.size())); 694 695 moveAppendSections(JDS.Swift5TypesSections, JDS.Swift5TypesSectionsNew); 696 return Error::success(); 697 } 698 699 Error MachOPlatformRuntimeState::runModInits(JITDylibState &JDS) { 700 701 for (const auto &ModInits : JDS.ModInitsSectionsNew) { 702 for (void (*Init)() : ModInits) 703 (*Init)(); 704 } 705 706 moveAppendSections(JDS.ModInitsSections, JDS.ModInitsSectionsNew); 707 return Error::success(); 708 } 709 710 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(string_view Path, 711 int Mode) { 712 // Try to find JITDylib state by name. 713 auto *JDS = getJITDylibStateByName(Path); 714 715 if (!JDS) 716 return make_error<StringError>("No registered JTIDylib for path " + 717 std::string(Path.data(), Path.size())); 718 719 // If this JITDylib is unsealed, or this is the first dlopen then run 720 // full dlopen path (update deps, push and run initializers, update ref 721 // counts on all JITDylibs in the dep tree). 722 if (!JDS->referenced() || !JDS->Sealed) { 723 if (auto Err = dlopenFull(*JDS)) 724 return std::move(Err); 725 } 726 727 // Bump the ref-count on this dylib. 728 ++JDS->DlRefCount; 729 730 // Return the header address. 731 return JDS->Header; 732 } 733 734 Error MachOPlatformRuntimeState::dlopenFull(JITDylibState &JDS) { 735 // Call back to the JIT to push the initializers. 736 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap())); 737 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>( 738 SPSExecutorAddr)>::call(&__orc_rt_macho_push_initializers_tag, 739 DepInfo, ExecutorAddr::fromPtr(JDS.Header))) 740 return Err; 741 if (!DepInfo) 742 return DepInfo.takeError(); 743 744 if (auto Err = dlopenInitialize(JDS, *DepInfo)) 745 return Err; 746 747 if (!DepInfo->empty()) { 748 ORC_RT_DEBUG({ 749 printdbg("Unrecognized dep-info key headers in dlopen of %s\n", 750 JDS.Name.c_str()); 751 }); 752 std::ostringstream ErrStream; 753 ErrStream << "Encountered unrecognized dep-info key headers " 754 "while processing dlopen of " 755 << JDS.Name; 756 return make_error<StringError>(ErrStream.str()); 757 } 758 759 return Error::success(); 760 } 761 762 Error MachOPlatformRuntimeState::dlopenInitialize( 763 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo) { 764 ORC_RT_DEBUG({ 765 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n", 766 JDS.Name.c_str()); 767 }); 768 769 // If the header is not present in the dep map then assume that we 770 // already processed it earlier in the dlopenInitialize traversal and 771 // return. 772 // TODO: Keep a visited set instead so that we can error out on missing 773 // entries? 774 auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header)); 775 if (I == DepInfo.end()) 776 return Error::success(); 777 778 auto DI = std::move(I->second); 779 DepInfo.erase(I); 780 781 // We don't need to re-initialize sealed JITDylibs that have already been 782 // initialized. Just check that their dep-map entry is empty as expected. 783 if (JDS.Sealed) { 784 if (!DI.DepHeaders.empty()) { 785 std::ostringstream ErrStream; 786 ErrStream << "Sealed JITDylib " << JDS.Header 787 << " already has registered dependencies"; 788 return make_error<StringError>(ErrStream.str()); 789 } 790 if (JDS.referenced()) 791 return Error::success(); 792 } else 793 JDS.Sealed = DI.Sealed; 794 795 // This is an unsealed or newly sealed JITDylib. Run initializers. 796 std::vector<JITDylibState *> OldDeps; 797 std::swap(JDS.Deps, OldDeps); 798 JDS.Deps.reserve(DI.DepHeaders.size()); 799 for (auto DepHeaderAddr : DI.DepHeaders) { 800 auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>()); 801 if (!DepJDS) { 802 std::ostringstream ErrStream; 803 ErrStream << "Encountered unrecognized dep header " 804 << DepHeaderAddr.toPtr<void *>() << " while initializing " 805 << JDS.Name; 806 return make_error<StringError>(ErrStream.str()); 807 } 808 ++DepJDS->LinkedAgainstRefCount; 809 if (auto Err = dlopenInitialize(*DepJDS, DepInfo)) 810 return Err; 811 } 812 813 // Initialize this JITDylib. 814 if (auto Err = registerObjCSelectors(JDS)) 815 return Err; 816 if (auto Err = registerObjCClasses(JDS)) 817 return Err; 818 if (auto Err = registerSwift5Protocols(JDS)) 819 return Err; 820 if (auto Err = registerSwift5ProtocolConformances(JDS)) 821 return Err; 822 if (auto Err = registerSwift5Types(JDS)) 823 return Err; 824 if (auto Err = runModInits(JDS)) 825 return Err; 826 827 // Decrement old deps. 828 // FIXME: We should probably continue and just report deinitialize errors 829 // here. 830 for (auto *DepJDS : OldDeps) { 831 --DepJDS->LinkedAgainstRefCount; 832 if (!DepJDS->referenced()) 833 if (auto Err = dlcloseDeinitialize(*DepJDS)) 834 return Err; 835 } 836 837 return Error::success(); 838 } 839 840 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) { 841 // Try to find JITDylib state by header. 842 auto *JDS = getJITDylibStateByHeader(DSOHandle); 843 844 if (!JDS) { 845 std::ostringstream ErrStream; 846 ErrStream << "No registered JITDylib for " << DSOHandle; 847 return make_error<StringError>(ErrStream.str()); 848 } 849 850 // Bump the ref-count. 851 --JDS->DlRefCount; 852 853 if (!JDS->referenced()) 854 return dlcloseDeinitialize(*JDS); 855 856 return Error::success(); 857 } 858 859 Error MachOPlatformRuntimeState::dlcloseDeinitialize(JITDylibState &JDS) { 860 861 ORC_RT_DEBUG({ 862 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n", 863 JDS.Name.c_str()); 864 }); 865 866 runAtExits(JDS); 867 868 // Reset mod-inits 869 moveAppendSections(JDS.ModInitsSections, JDS.ModInitsSectionsNew); 870 JDS.ModInitsSectionsNew = std::move(JDS.ModInitsSections); 871 872 // Deinitialize any dependencies. 873 for (auto *DepJDS : JDS.Deps) { 874 --DepJDS->LinkedAgainstRefCount; 875 if (!DepJDS->referenced()) 876 if (auto Err = dlcloseDeinitialize(*DepJDS)) 877 return Err; 878 } 879 880 return Error::success(); 881 } 882 883 class MachOPlatformRuntimeTLVManager { 884 public: 885 void *getInstance(const char *ThreadData); 886 887 private: 888 std::unordered_map<const char *, char *> Instances; 889 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections; 890 }; 891 892 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) { 893 auto I = Instances.find(ThreadData); 894 if (I != Instances.end()) 895 return I->second; 896 897 auto TDS = 898 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData); 899 if (!TDS) { 900 __orc_rt_log_error(toString(TDS.takeError()).c_str()); 901 return nullptr; 902 } 903 904 auto &Allocated = AllocatedSections[TDS->first]; 905 if (!Allocated) { 906 Allocated = std::make_unique<char[]>(TDS->second); 907 memcpy(Allocated.get(), TDS->first, TDS->second); 908 } 909 910 size_t ThreadDataDelta = ThreadData - TDS->first; 911 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds"); 912 913 char *Instance = Allocated.get() + ThreadDataDelta; 914 Instances[ThreadData] = Instance; 915 return Instance; 916 } 917 918 void destroyMachOTLVMgr(void *MachOTLVMgr) { 919 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr); 920 } 921 922 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) { 923 for (auto &WFC : WFCs) 924 if (auto Err = WFC.runWithSPSRet<void>()) 925 return Err; 926 return Error::success(); 927 } 928 929 } // end anonymous namespace 930 931 //------------------------------------------------------------------------------ 932 // JIT entry points 933 //------------------------------------------------------------------------------ 934 935 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 936 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) { 937 MachOPlatformRuntimeState::initialize(); 938 return WrapperFunctionResult().release(); 939 } 940 941 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 942 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) { 943 MachOPlatformRuntimeState::destroy(); 944 return WrapperFunctionResult().release(); 945 } 946 947 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 948 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) { 949 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle( 950 ArgData, ArgSize, 951 [](std::string &Name, ExecutorAddr HeaderAddr) { 952 return MachOPlatformRuntimeState::get().registerJITDylib( 953 std::move(Name), HeaderAddr.toPtr<void *>()); 954 }) 955 .release(); 956 } 957 958 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 959 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) { 960 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle( 961 ArgData, ArgSize, 962 [](ExecutorAddr HeaderAddr) { 963 return MachOPlatformRuntimeState::get().deregisterJITDylib( 964 HeaderAddr.toPtr<void *>()); 965 }) 966 .release(); 967 } 968 969 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 970 __orc_rt_macho_register_object_platform_sections(char *ArgData, 971 size_t ArgSize) { 972 return WrapperFunction<SPSError(SPSExecutorAddr, 973 SPSMachOObjectPlatformSectionsMap)>:: 974 handle(ArgData, ArgSize, 975 [](ExecutorAddr HeaderAddr, 976 std::vector<std::pair<string_view, ExecutorAddrRange>> &Secs) { 977 return MachOPlatformRuntimeState::get() 978 .registerObjectPlatformSections(HeaderAddr, std::move(Secs)); 979 }) 980 .release(); 981 } 982 983 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 984 __orc_rt_macho_deregister_object_platform_sections(char *ArgData, 985 size_t ArgSize) { 986 return WrapperFunction<SPSError(SPSExecutorAddr, 987 SPSMachOObjectPlatformSectionsMap)>:: 988 handle(ArgData, ArgSize, 989 [](ExecutorAddr HeaderAddr, 990 std::vector<std::pair<string_view, ExecutorAddrRange>> &Secs) { 991 return MachOPlatformRuntimeState::get() 992 .deregisterObjectPlatformSections(HeaderAddr, 993 std::move(Secs)); 994 }) 995 .release(); 996 } 997 998 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 999 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) { 1000 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle( 1001 ArgData, ArgSize, runWrapperFunctionCalls) 1002 .release(); 1003 } 1004 1005 //------------------------------------------------------------------------------ 1006 // TLV support 1007 //------------------------------------------------------------------------------ 1008 1009 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) { 1010 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>( 1011 pthread_getspecific(D->Key)); 1012 if (!TLVMgr) { 1013 TLVMgr = new MachOPlatformRuntimeTLVManager(); 1014 if (pthread_setspecific(D->Key, TLVMgr)) { 1015 __orc_rt_log_error("Call to pthread_setspecific failed"); 1016 return nullptr; 1017 } 1018 } 1019 1020 return TLVMgr->getInstance( 1021 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress))); 1022 } 1023 1024 ORC_RT_INTERFACE __orc_rt_CWrapperFunctionResult 1025 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) { 1026 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle( 1027 ArgData, ArgSize, 1028 []() -> Expected<uint64_t> { 1029 pthread_key_t Key; 1030 if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) { 1031 __orc_rt_log_error("Call to pthread_key_create failed"); 1032 return make_error<StringError>(strerror(Err)); 1033 } 1034 return static_cast<uint64_t>(Key); 1035 }) 1036 .release(); 1037 } 1038 1039 //------------------------------------------------------------------------------ 1040 // cxa_atexit support 1041 //------------------------------------------------------------------------------ 1042 1043 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg, 1044 void *dso_handle) { 1045 return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle); 1046 } 1047 1048 void __orc_rt_macho_cxa_finalize(void *dso_handle) { 1049 MachOPlatformRuntimeState::get().runAtExits(dso_handle); 1050 } 1051 1052 //------------------------------------------------------------------------------ 1053 // JIT'd dlfcn alternatives. 1054 //------------------------------------------------------------------------------ 1055 1056 const char *__orc_rt_macho_jit_dlerror() { 1057 return MachOPlatformRuntimeState::get().dlerror(); 1058 } 1059 1060 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) { 1061 return MachOPlatformRuntimeState::get().dlopen(path, mode); 1062 } 1063 1064 int __orc_rt_macho_jit_dlclose(void *dso_handle) { 1065 return MachOPlatformRuntimeState::get().dlclose(dso_handle); 1066 } 1067 1068 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) { 1069 return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol); 1070 } 1071 1072 //------------------------------------------------------------------------------ 1073 // MachO Run Program 1074 //------------------------------------------------------------------------------ 1075 1076 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName, 1077 const char *EntrySymbolName, 1078 int argc, char *argv[]) { 1079 using MainTy = int (*)(int, char *[]); 1080 1081 void *H = __orc_rt_macho_jit_dlopen(JITDylibName, 1082 __orc_rt::macho::ORC_RT_RTLD_LAZY); 1083 if (!H) { 1084 __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); 1085 return -1; 1086 } 1087 1088 auto *Main = 1089 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName)); 1090 1091 if (!Main) { 1092 __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); 1093 return -1; 1094 } 1095 1096 int Result = Main(argc, argv); 1097 1098 if (__orc_rt_macho_jit_dlclose(H) == -1) 1099 __orc_rt_log_error(__orc_rt_macho_jit_dlerror()); 1100 1101 return Result; 1102 } 1103