1 //===- macho_platform.cpp -------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains code required to load the rest of the MachO runtime.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "macho_platform.h"
14 #include "bitmask_enum.h"
15 #include "common.h"
16 #include "debug.h"
17 #include "error.h"
18 #include "interval_map.h"
19 #include "jit_dispatch.h"
20 #include "record_section_tracker.h"
21 #include "wrapper_function_utils.h"
22
23 #include <algorithm>
24 #include <ios>
25 #include <map>
26 #include <mutex>
27 #include <sstream>
28 #include <string_view>
29 #include <unordered_map>
30 #include <unordered_set>
31 #include <vector>
32
33 #define DEBUG_TYPE "macho_platform"
34
35 using namespace orc_rt;
36 using namespace orc_rt::macho;
37
38 // Declare function tags for functions in the JIT process.
39 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_initializers_tag)
40 ORC_RT_JIT_DISPATCH_TAG(__orc_rt_macho_push_symbols_tag)
41
42 struct objc_image_info;
43 struct mach_header;
44
45 // Objective-C registration functions.
46 // These are weakly imported. If the Objective-C runtime has not been loaded
47 // then code containing Objective-C sections will generate an error.
48 extern "C" void
49 _objc_map_images(unsigned count, const char *const paths[],
50 const mach_header *const mhdrs[]) ORC_RT_WEAK_IMPORT;
51
52 extern "C" void _objc_load_image(const char *path,
53 const mach_header *mh) ORC_RT_WEAK_IMPORT;
54
55 // Libunwind prototypes.
56 struct unw_dynamic_unwind_sections {
57 uintptr_t dso_base;
58 uintptr_t dwarf_section;
59 size_t dwarf_section_length;
60 uintptr_t compact_unwind_section;
61 size_t compact_unwind_section_length;
62 };
63
64 typedef int (*unw_find_dynamic_unwind_sections)(
65 uintptr_t addr, struct unw_dynamic_unwind_sections *info);
66
67 extern "C" int __unw_add_find_dynamic_unwind_sections(
68 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
69 ORC_RT_WEAK_IMPORT;
70
71 extern "C" int __unw_remove_find_dynamic_unwind_sections(
72 unw_find_dynamic_unwind_sections find_dynamic_unwind_sections)
73 ORC_RT_WEAK_IMPORT;
74
75 namespace {
76
77 struct MachOJITDylibDepInfo {
78 bool Sealed = false;
79 std::vector<ExecutorAddr> DepHeaders;
80 };
81
82 using MachOJITDylibDepInfoMap =
83 std::unordered_map<ExecutorAddr, MachOJITDylibDepInfo>;
84
85 } // anonymous namespace
86
87 namespace orc_rt {
88
89 using SPSMachOObjectPlatformSectionsMap =
90 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
91
92 using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>;
93
94 using SPSMachOJITDylibDepInfoMap =
95 SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>;
96
97 template <>
98 class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOJITDylibDepInfo> {
99 public:
size(const MachOJITDylibDepInfo & JDI)100 static size_t size(const MachOJITDylibDepInfo &JDI) {
101 return SPSMachOJITDylibDepInfo::AsArgList::size(JDI.Sealed, JDI.DepHeaders);
102 }
103
serialize(SPSOutputBuffer & OB,const MachOJITDylibDepInfo & JDI)104 static bool serialize(SPSOutputBuffer &OB, const MachOJITDylibDepInfo &JDI) {
105 return SPSMachOJITDylibDepInfo::AsArgList::serialize(OB, JDI.Sealed,
106 JDI.DepHeaders);
107 }
108
deserialize(SPSInputBuffer & IB,MachOJITDylibDepInfo & JDI)109 static bool deserialize(SPSInputBuffer &IB, MachOJITDylibDepInfo &JDI) {
110 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, JDI.Sealed,
111 JDI.DepHeaders);
112 }
113 };
114
115 struct UnwindSectionInfo {
116 std::vector<ExecutorAddrRange> CodeRanges;
117 ExecutorAddrRange DwarfSection;
118 ExecutorAddrRange CompactUnwindSection;
119 };
120
121 using SPSUnwindSectionInfo =
122 SPSTuple<SPSSequence<SPSExecutorAddrRange>, SPSExecutorAddrRange,
123 SPSExecutorAddrRange>;
124
125 template <>
126 class SPSSerializationTraits<SPSUnwindSectionInfo, UnwindSectionInfo> {
127 public:
size(const UnwindSectionInfo & USI)128 static size_t size(const UnwindSectionInfo &USI) {
129 return SPSUnwindSectionInfo::AsArgList::size(
130 USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
131 }
132
serialize(SPSOutputBuffer & OB,const UnwindSectionInfo & USI)133 static bool serialize(SPSOutputBuffer &OB, const UnwindSectionInfo &USI) {
134 return SPSUnwindSectionInfo::AsArgList::serialize(
135 OB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
136 }
137
deserialize(SPSInputBuffer & IB,UnwindSectionInfo & USI)138 static bool deserialize(SPSInputBuffer &IB, UnwindSectionInfo &USI) {
139 return SPSUnwindSectionInfo::AsArgList::deserialize(
140 IB, USI.CodeRanges, USI.DwarfSection, USI.CompactUnwindSection);
141 }
142 };
143
144 } // namespace orc_rt
145
146 namespace {
147 struct TLVDescriptor {
148 void *(*Thunk)(TLVDescriptor *) = nullptr;
149 unsigned long Key = 0;
150 unsigned long DataAddress = 0;
151 };
152
153 class MachOPlatformRuntimeState {
154 public:
155 // Used internally by MachOPlatformRuntimeState, but made public to enable
156 // serialization.
157 enum class MachOExecutorSymbolFlags : uint8_t {
158 None = 0,
159 Weak = 1U << 0,
160 Callable = 1U << 1,
161 ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable)
162 };
163
164 private:
165 struct AtExitEntry {
166 void (*Func)(void *);
167 void *Arg;
168 };
169
170 using AtExitsVector = std::vector<AtExitEntry>;
171
172 struct UnwindSections {
UnwindSections__anon3d23d5020211::MachOPlatformRuntimeState::UnwindSections173 UnwindSections(const UnwindSectionInfo &USI)
174 : DwarfSection(USI.DwarfSection.toSpan<char>()),
175 CompactUnwindSection(USI.CompactUnwindSection.toSpan<char>()) {}
176
177 span<char> DwarfSection;
178 span<char> CompactUnwindSection;
179 };
180
181 using UnwindSectionsMap =
182 IntervalMap<char *, UnwindSections, IntervalCoalescing::Disabled>;
183
184 struct JITDylibState {
185
186 using SymbolTableMap =
187 std::unordered_map<std::string_view,
188 std::pair<ExecutorAddr, MachOExecutorSymbolFlags>>;
189
190 std::string Name;
191 void *Header = nullptr;
192 bool Sealed = false;
193 size_t LinkedAgainstRefCount = 0;
194 size_t DlRefCount = 0;
195 SymbolTableMap SymbolTable;
196 std::vector<JITDylibState *> Deps;
197 AtExitsVector AtExits;
198 const objc_image_info *ObjCImageInfo = nullptr;
199 std::unordered_map<void *, std::vector<char>> DataSectionContent;
200 std::unordered_map<void *, size_t> ZeroInitRanges;
201 UnwindSectionsMap UnwindSections;
202 RecordSectionsTracker<void (*)()> ModInitsSections;
203 RecordSectionsTracker<char> ObjCRuntimeRegistrationObjects;
204
referenced__anon3d23d5020211::MachOPlatformRuntimeState::JITDylibState205 bool referenced() const {
206 return LinkedAgainstRefCount != 0 || DlRefCount != 0;
207 }
208 };
209
210 public:
211 static Error create();
212 static MachOPlatformRuntimeState &get();
213 static Error destroy();
214
215 MachOPlatformRuntimeState() = default;
216
217 // Delete copy and move constructors.
218 MachOPlatformRuntimeState(const MachOPlatformRuntimeState &) = delete;
219 MachOPlatformRuntimeState &
220 operator=(const MachOPlatformRuntimeState &) = delete;
221 MachOPlatformRuntimeState(MachOPlatformRuntimeState &&) = delete;
222 MachOPlatformRuntimeState &operator=(MachOPlatformRuntimeState &&) = delete;
223
224 Error initialize();
225 Error shutdown();
226
227 Error registerJITDylib(std::string Name, void *Header);
228 Error deregisterJITDylib(void *Header);
229 Error registerThreadDataSection(span<const char> ThreadDataSection);
230 Error deregisterThreadDataSection(span<const char> ThreadDataSection);
231 Error registerObjectSymbolTable(
232 ExecutorAddr HeaderAddr,
233 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
234 MachOExecutorSymbolFlags>> &Entries);
235 Error deregisterObjectSymbolTable(
236 ExecutorAddr HeaderAddr,
237 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
238 MachOExecutorSymbolFlags>> &Entries);
239 Error registerObjectPlatformSections(
240 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
241 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
242 Error deregisterObjectPlatformSections(
243 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindSections,
244 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs);
245
246 const char *dlerror();
247 void *dlopen(std::string_view Name, int Mode);
248 int dlupdate(void *DSOHandle);
249 int dlclose(void *DSOHandle);
250 void *dlsym(void *DSOHandle, const char *Symbol);
251
252 int registerAtExit(void (*F)(void *), void *Arg, void *DSOHandle);
253 void runAtExits(std::unique_lock<std::mutex> &JDStatesLock,
254 JITDylibState &JDS);
255 void runAtExits(void *DSOHandle);
256
257 /// Returns the base address of the section containing ThreadData.
258 Expected<std::pair<const char *, size_t>>
259 getThreadDataSectionFor(const char *ThreadData);
260
261 private:
262 JITDylibState *getJITDylibStateByHeader(void *DSOHandle);
263 JITDylibState *getJITDylibStateByName(std::string_view Path);
264
265 /// Requests materialization of the given symbols. For each pair, the bool
266 /// element indicates whether the symbol is required (true) or weakly
267 /// referenced (false).
268 Error requestPushSymbols(JITDylibState &JDS,
269 span<std::pair<std::string_view, bool>> Symbols);
270
271 /// Attempts to look up the given symbols locally, requesting a push from the
272 /// remote if they're not found. Results are written to the Result span, which
273 /// must have the same size as the Symbols span.
274 Error
275 lookupSymbols(JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
276 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
277 span<std::pair<std::string_view, bool>> Symbols);
278
279 bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info);
280
281 static int findDynamicUnwindSections(uintptr_t addr,
282 unw_dynamic_unwind_sections *info);
283 static Error registerEHFrames(span<const char> EHFrameSection);
284 static Error deregisterEHFrames(span<const char> EHFrameSection);
285
286 static Error
287 registerObjCRegistrationObjects(std::unique_lock<std::mutex> &JDStatesLock,
288 JITDylibState &JDS);
289 static Error runModInits(std::unique_lock<std::mutex> &JDStatesLock,
290 JITDylibState &JDS);
291
292 Expected<void *> dlopenImpl(std::string_view Path, int Mode);
293 Error dlopenFull(std::unique_lock<std::mutex> &JDStatesLock,
294 JITDylibState &JDS);
295 Error dlopenInitialize(std::unique_lock<std::mutex> &JDStatesLock,
296 JITDylibState &JDS, MachOJITDylibDepInfoMap &DepInfo);
297
298 Error dlupdateImpl(void *DSOHandle);
299 Error dlupdateFull(std::unique_lock<std::mutex> &JDStatesLock,
300 JITDylibState &JDS);
301 Error dlupdateInitialize(std::unique_lock<std::mutex> &JDStatesLock,
302 JITDylibState &JDS);
303
304 Error dlcloseImpl(void *DSOHandle);
305 Error dlcloseDeinitialize(std::unique_lock<std::mutex> &JDStatesLock,
306 JITDylibState &JDS);
307
308 static MachOPlatformRuntimeState *MOPS;
309
310 bool UseCallbackStyleUnwindInfo = false;
311
312 // FIXME: Move to thread-state.
313 std::string DLFcnError;
314
315 // APIMutex guards against concurrent entry into key "dyld" API functions
316 // (e.g. dlopen, dlclose).
317 std::recursive_mutex DyldAPIMutex;
318
319 // JDStatesMutex guards the data structures that hold JITDylib state.
320 std::mutex JDStatesMutex;
321 std::unordered_map<void *, JITDylibState> JDStates;
322 std::unordered_map<std::string_view, void *> JDNameToHeader;
323
324 // ThreadDataSectionsMutex guards thread local data section state.
325 std::mutex ThreadDataSectionsMutex;
326 std::map<const char *, size_t> ThreadDataSections;
327 };
328
329 } // anonymous namespace
330
331 namespace orc_rt {
332
333 class SPSMachOExecutorSymbolFlags;
334
335 template <>
336 class SPSSerializationTraits<
337 SPSMachOExecutorSymbolFlags,
338 MachOPlatformRuntimeState::MachOExecutorSymbolFlags> {
339 private:
340 using UT = std::underlying_type_t<
341 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>;
342
343 public:
344 static size_t
size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags & SF)345 size(const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
346 return sizeof(UT);
347 }
348
349 static bool
serialize(SPSOutputBuffer & OB,const MachOPlatformRuntimeState::MachOExecutorSymbolFlags & SF)350 serialize(SPSOutputBuffer &OB,
351 const MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
352 return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF));
353 }
354
355 static bool
deserialize(SPSInputBuffer & IB,MachOPlatformRuntimeState::MachOExecutorSymbolFlags & SF)356 deserialize(SPSInputBuffer &IB,
357 MachOPlatformRuntimeState::MachOExecutorSymbolFlags &SF) {
358 UT Tmp;
359 if (!SPSArgList<UT>::deserialize(IB, Tmp))
360 return false;
361 SF = static_cast<MachOPlatformRuntimeState::MachOExecutorSymbolFlags>(Tmp);
362 return true;
363 }
364 };
365
366 } // namespace orc_rt
367
368 namespace {
369
370 MachOPlatformRuntimeState *MachOPlatformRuntimeState::MOPS = nullptr;
371
create()372 Error MachOPlatformRuntimeState::create() {
373 assert(!MOPS && "MachOPlatformRuntimeState should be null");
374 MOPS = new MachOPlatformRuntimeState();
375 return MOPS->initialize();
376 }
377
get()378 MachOPlatformRuntimeState &MachOPlatformRuntimeState::get() {
379 assert(MOPS && "MachOPlatformRuntimeState not initialized");
380 return *MOPS;
381 }
382
destroy()383 Error MachOPlatformRuntimeState::destroy() {
384 assert(MOPS && "MachOPlatformRuntimeState not initialized");
385 auto Err = MOPS->shutdown();
386 delete MOPS;
387 return Err;
388 }
389
initialize()390 Error MachOPlatformRuntimeState::initialize() {
391 UseCallbackStyleUnwindInfo = __unw_add_find_dynamic_unwind_sections &&
392 __unw_remove_find_dynamic_unwind_sections;
393 if (UseCallbackStyleUnwindInfo) {
394 ORC_RT_DEBUG({
395 printdbg("__unw_add/remove_find_dynamic_unwind_sections available."
396 " Using callback-based frame info lookup.\n");
397 });
398 if (__unw_add_find_dynamic_unwind_sections(&findDynamicUnwindSections))
399 return make_error<StringError>(
400 "Could not register findDynamicUnwindSections");
401 } else {
402 ORC_RT_DEBUG({
403 printdbg("__unw_add/remove_find_dynamic_unwind_sections not available."
404 " Using classic frame info registration.\n");
405 });
406 }
407 return Error::success();
408 }
409
shutdown()410 Error MachOPlatformRuntimeState::shutdown() {
411 if (UseCallbackStyleUnwindInfo) {
412 if (__unw_remove_find_dynamic_unwind_sections(&findDynamicUnwindSections)) {
413 ORC_RT_DEBUG(
414 { printdbg("__unw_remove_find_dynamic_unwind_sections failed.\n"); });
415 }
416 }
417 return Error::success();
418 }
419
registerJITDylib(std::string Name,void * Header)420 Error MachOPlatformRuntimeState::registerJITDylib(std::string Name,
421 void *Header) {
422 ORC_RT_DEBUG({
423 printdbg("Registering JITDylib %s: Header = %p\n", Name.c_str(), Header);
424 });
425 std::lock_guard<std::mutex> Lock(JDStatesMutex);
426 if (JDStates.count(Header)) {
427 std::ostringstream ErrStream;
428 ErrStream << "Duplicate JITDylib registration for header " << Header
429 << " (name = " << Name << ")";
430 return make_error<StringError>(ErrStream.str());
431 }
432 if (JDNameToHeader.count(Name)) {
433 std::ostringstream ErrStream;
434 ErrStream << "Duplicate JITDylib registration for header " << Header
435 << " (header = " << Header << ")";
436 return make_error<StringError>(ErrStream.str());
437 }
438
439 auto &JDS = JDStates[Header];
440 JDS.Name = std::move(Name);
441 JDS.Header = Header;
442 JDNameToHeader[JDS.Name] = Header;
443 return Error::success();
444 }
445
deregisterJITDylib(void * Header)446 Error MachOPlatformRuntimeState::deregisterJITDylib(void *Header) {
447 std::lock_guard<std::mutex> Lock(JDStatesMutex);
448 auto I = JDStates.find(Header);
449 if (I == JDStates.end()) {
450 std::ostringstream ErrStream;
451 ErrStream << "Attempted to deregister unrecognized header " << Header;
452 return make_error<StringError>(ErrStream.str());
453 }
454
455 // Remove std::string construction once we can use C++20.
456 auto J = JDNameToHeader.find(
457 std::string(I->second.Name.data(), I->second.Name.size()));
458 assert(J != JDNameToHeader.end() &&
459 "Missing JDNameToHeader entry for JITDylib");
460
461 ORC_RT_DEBUG({
462 printdbg("Deregistering JITDylib %s: Header = %p\n", I->second.Name.c_str(),
463 Header);
464 });
465
466 JDNameToHeader.erase(J);
467 JDStates.erase(I);
468 return Error::success();
469 }
470
registerThreadDataSection(span<const char> ThreadDataSection)471 Error MachOPlatformRuntimeState::registerThreadDataSection(
472 span<const char> ThreadDataSection) {
473 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
474 auto I = ThreadDataSections.upper_bound(ThreadDataSection.data());
475 if (I != ThreadDataSections.begin()) {
476 auto J = std::prev(I);
477 if (J->first + J->second > ThreadDataSection.data())
478 return make_error<StringError>("Overlapping __thread_data sections");
479 }
480 ThreadDataSections.insert(
481 I, std::make_pair(ThreadDataSection.data(), ThreadDataSection.size()));
482 return Error::success();
483 }
484
deregisterThreadDataSection(span<const char> ThreadDataSection)485 Error MachOPlatformRuntimeState::deregisterThreadDataSection(
486 span<const char> ThreadDataSection) {
487 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
488 auto I = ThreadDataSections.find(ThreadDataSection.data());
489 if (I == ThreadDataSections.end())
490 return make_error<StringError>("Attempt to deregister unknown thread data "
491 "section");
492 ThreadDataSections.erase(I);
493 return Error::success();
494 }
495
registerObjectSymbolTable(ExecutorAddr HeaderAddr,const std::vector<std::tuple<ExecutorAddr,ExecutorAddr,MachOExecutorSymbolFlags>> & Entries)496 Error MachOPlatformRuntimeState::registerObjectSymbolTable(
497 ExecutorAddr HeaderAddr,
498 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
499 MachOExecutorSymbolFlags>> &Entries) {
500
501 std::lock_guard<std::mutex> Lock(JDStatesMutex);
502 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
503 if (!JDS) {
504 std::ostringstream ErrStream;
505 ErrStream << "Could not register object platform sections for "
506 "unrecognized header "
507 << HeaderAddr.toPtr<void *>();
508 return make_error<StringError>(ErrStream.str());
509 }
510
511 for (auto &[NameAddr, SymAddr, Flags] : Entries)
512 JDS->SymbolTable[NameAddr.toPtr<const char *>()] = {SymAddr, Flags};
513
514 return Error::success();
515 }
516
deregisterObjectSymbolTable(ExecutorAddr HeaderAddr,const std::vector<std::tuple<ExecutorAddr,ExecutorAddr,MachOExecutorSymbolFlags>> & Entries)517 Error MachOPlatformRuntimeState::deregisterObjectSymbolTable(
518 ExecutorAddr HeaderAddr,
519 const std::vector<std::tuple<ExecutorAddr, ExecutorAddr,
520 MachOExecutorSymbolFlags>> &Entries) {
521
522 std::lock_guard<std::mutex> Lock(JDStatesMutex);
523 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
524 if (!JDS) {
525 std::ostringstream ErrStream;
526 ErrStream << "Could not register object platform sections for "
527 "unrecognized header "
528 << HeaderAddr.toPtr<void *>();
529 return make_error<StringError>(ErrStream.str());
530 }
531
532 for (auto &[NameAddr, SymAddr, Flags] : Entries)
533 JDS->SymbolTable.erase(NameAddr.toPtr<const char *>());
534
535 return Error::success();
536 }
537
registerObjectPlatformSections(ExecutorAddr HeaderAddr,std::optional<UnwindSectionInfo> UnwindInfo,std::vector<std::pair<std::string_view,ExecutorAddrRange>> Secs)538 Error MachOPlatformRuntimeState::registerObjectPlatformSections(
539 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
540 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
541
542 // FIXME: Reject platform section registration after the JITDylib is
543 // sealed?
544
545 ORC_RT_DEBUG({
546 printdbg("MachOPlatform: Registering object sections for %p.\n",
547 HeaderAddr.toPtr<void *>());
548 });
549
550 std::lock_guard<std::mutex> Lock(JDStatesMutex);
551 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
552 if (!JDS) {
553 std::ostringstream ErrStream;
554 ErrStream << "Could not register object platform sections for "
555 "unrecognized header "
556 << HeaderAddr.toPtr<void *>();
557 return make_error<StringError>(ErrStream.str());
558 }
559
560 ORC_RT_DEBUG({
561 printdbg(" UnwindInfo: %s, UseCallbackStyleUnwindInfo: %s\n",
562 UnwindInfo ? "true" : "false",
563 UseCallbackStyleUnwindInfo ? "true" : "false");
564 });
565
566 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
567 ORC_RT_DEBUG({
568 printdbg(" Registering new-style unwind info for:\n"
569 " DWARF: %p -- %p\n"
570 " Compact-unwind: %p -- %p\n"
571 " for:\n",
572 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
573 UnwindInfo->DwarfSection.End.toPtr<void *>(),
574 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
575 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
576 });
577 for (auto &CodeRange : UnwindInfo->CodeRanges) {
578 JDS->UnwindSections.insert(CodeRange.Start.toPtr<char *>(),
579 CodeRange.End.toPtr<char *>(), *UnwindInfo);
580 ORC_RT_DEBUG({
581 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
582 CodeRange.End.toPtr<void *>());
583 });
584 }
585 }
586
587 for (auto &KV : Secs) {
588 // FIXME: Validate section ranges?
589 if (KV.first == "__TEXT,__eh_frame") {
590 if (!UseCallbackStyleUnwindInfo) {
591 // Use classic libunwind registration.
592 if (auto Err = registerEHFrames(KV.second.toSpan<const char>()))
593 return Err;
594 }
595 } else if (KV.first == "__DATA,__data") {
596 assert(!JDS->DataSectionContent.count(KV.second.Start.toPtr<char *>()) &&
597 "Address already registered.");
598 auto S = KV.second.toSpan<char>();
599 JDS->DataSectionContent[KV.second.Start.toPtr<char *>()] =
600 std::vector<char>(S.begin(), S.end());
601 } else if (KV.first == "__DATA,__common") {
602 JDS->ZeroInitRanges[KV.second.Start.toPtr<char *>()] = KV.second.size();
603 } else if (KV.first == "__DATA,__thread_data") {
604 if (auto Err = registerThreadDataSection(KV.second.toSpan<const char>()))
605 return Err;
606 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
607 JDS->ObjCRuntimeRegistrationObjects.add(KV.second.toSpan<char>());
608 else if (KV.first == "__DATA,__mod_init_func")
609 JDS->ModInitsSections.add(KV.second.toSpan<void (*)()>());
610 else {
611 // Should this be a warning instead?
612 return make_error<StringError>(
613 "Encountered unexpected section " +
614 std::string(KV.first.data(), KV.first.size()) +
615 " while registering object platform sections");
616 }
617 }
618
619 return Error::success();
620 }
621
deregisterObjectPlatformSections(ExecutorAddr HeaderAddr,std::optional<UnwindSectionInfo> UnwindInfo,std::vector<std::pair<std::string_view,ExecutorAddrRange>> Secs)622 Error MachOPlatformRuntimeState::deregisterObjectPlatformSections(
623 ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> UnwindInfo,
624 std::vector<std::pair<std::string_view, ExecutorAddrRange>> Secs) {
625 // TODO: Make this more efficient? (maybe unnecessary if removal is rare?)
626 // TODO: Add a JITDylib prepare-for-teardown operation that clears all
627 // registered sections, causing this function to take the fast-path.
628 ORC_RT_DEBUG({
629 printdbg("MachOPlatform: Deregistering object sections for %p.\n",
630 HeaderAddr.toPtr<void *>());
631 });
632
633 std::lock_guard<std::mutex> Lock(JDStatesMutex);
634 auto *JDS = getJITDylibStateByHeader(HeaderAddr.toPtr<void *>());
635 if (!JDS) {
636 std::ostringstream ErrStream;
637 ErrStream << "Could not register object platform sections for unrecognized "
638 "header "
639 << HeaderAddr.toPtr<void *>();
640 return make_error<StringError>(ErrStream.str());
641 }
642
643 // FIXME: Implement faster-path by returning immediately if JDS is being
644 // torn down entirely?
645
646 // TODO: Make library permanent (i.e. not able to be dlclosed) if it contains
647 // any Swift or ObjC. Once this happens we can clear (and no longer record)
648 // data section content, as the library could never be re-initialized.
649
650 if (UnwindInfo && UseCallbackStyleUnwindInfo) {
651 ORC_RT_DEBUG({
652 printdbg(" Deregistering new-style unwind info for:\n"
653 " DWARF: %p -- %p\n"
654 " Compact-unwind: %p -- %p\n"
655 " for:\n",
656 UnwindInfo->DwarfSection.Start.toPtr<void *>(),
657 UnwindInfo->DwarfSection.End.toPtr<void *>(),
658 UnwindInfo->CompactUnwindSection.Start.toPtr<void *>(),
659 UnwindInfo->CompactUnwindSection.End.toPtr<void *>());
660 });
661 for (auto &CodeRange : UnwindInfo->CodeRanges) {
662 JDS->UnwindSections.erase(CodeRange.Start.toPtr<char *>(),
663 CodeRange.End.toPtr<char *>());
664 ORC_RT_DEBUG({
665 printdbg(" [ %p -- %p ]\n", CodeRange.Start.toPtr<void *>(),
666 CodeRange.End.toPtr<void *>());
667 });
668 }
669 }
670
671 for (auto &KV : Secs) {
672 // FIXME: Validate section ranges?
673 if (KV.first == "__TEXT,__eh_frame") {
674 if (!UseCallbackStyleUnwindInfo) {
675 // Use classic libunwind registration.
676 if (auto Err = deregisterEHFrames(KV.second.toSpan<const char>()))
677 return Err;
678 }
679 } else if (KV.first == "__DATA,__data") {
680 JDS->DataSectionContent.erase(KV.second.Start.toPtr<char *>());
681 } else if (KV.first == "__DATA,__common") {
682 JDS->ZeroInitRanges.erase(KV.second.Start.toPtr<char *>());
683 } else if (KV.first == "__DATA,__thread_data") {
684 if (auto Err =
685 deregisterThreadDataSection(KV.second.toSpan<const char>()))
686 return Err;
687 } else if (KV.first == "__llvm_jitlink_ObjCRuntimeRegistrationObject")
688 JDS->ObjCRuntimeRegistrationObjects.removeIfPresent(KV.second);
689 else if (KV.first == "__DATA,__mod_init_func")
690 JDS->ModInitsSections.removeIfPresent(KV.second);
691 else {
692 // Should this be a warning instead?
693 return make_error<StringError>(
694 "Encountered unexpected section " +
695 std::string(KV.first.data(), KV.first.size()) +
696 " while deregistering object platform sections");
697 }
698 }
699 return Error::success();
700 }
701
dlerror()702 const char *MachOPlatformRuntimeState::dlerror() { return DLFcnError.c_str(); }
703
dlopen(std::string_view Path,int Mode)704 void *MachOPlatformRuntimeState::dlopen(std::string_view Path, int Mode) {
705 ORC_RT_DEBUG({
706 std::string S(Path.data(), Path.size());
707 printdbg("MachOPlatform::dlopen(\"%s\")\n", S.c_str());
708 });
709 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
710 if (auto H = dlopenImpl(Path, Mode))
711 return *H;
712 else {
713 // FIXME: Make dlerror thread safe.
714 DLFcnError = toString(H.takeError());
715 return nullptr;
716 }
717 }
718
dlupdate(void * DSOHandle)719 int MachOPlatformRuntimeState::dlupdate(void *DSOHandle) {
720 ORC_RT_DEBUG({
721 std::string S;
722 printdbg("MachOPlatform::dlupdate(%p) (%s)\n", DSOHandle, S.c_str());
723 });
724 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
725 if (auto Err = dlupdateImpl(DSOHandle)) {
726 // FIXME: Make dlerror thread safe.
727 DLFcnError = toString(std::move(Err));
728 return -1;
729 }
730 return 0;
731 }
732
dlclose(void * DSOHandle)733 int MachOPlatformRuntimeState::dlclose(void *DSOHandle) {
734 ORC_RT_DEBUG({
735 auto *JDS = getJITDylibStateByHeader(DSOHandle);
736 std::string DylibName;
737 if (JDS) {
738 std::string S;
739 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle, S.c_str());
740 } else
741 printdbg("MachOPlatform::dlclose(%p) (%s)\n", DSOHandle,
742 "invalid handle");
743 });
744 std::lock_guard<std::recursive_mutex> Lock(DyldAPIMutex);
745 if (auto Err = dlcloseImpl(DSOHandle)) {
746 // FIXME: Make dlerror thread safe.
747 DLFcnError = toString(std::move(Err));
748 return -1;
749 }
750 return 0;
751 }
752
dlsym(void * DSOHandle,const char * Symbol)753 void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) {
754 std::unique_lock<std::mutex> Lock(JDStatesMutex);
755 auto *JDS = getJITDylibStateByHeader(DSOHandle);
756 if (!JDS) {
757 std::ostringstream ErrStream;
758 ErrStream << "In call to dlsym, unrecognized header address " << DSOHandle;
759 DLFcnError = ErrStream.str();
760 return nullptr;
761 }
762
763 std::string MangledName = std::string("_") + Symbol;
764 std::pair<std::string_view, bool> Lookup(MangledName, false);
765 std::pair<ExecutorAddr, MachOExecutorSymbolFlags> Result;
766
767 if (auto Err = lookupSymbols(*JDS, Lock, {&Result, 1}, {&Lookup, 1})) {
768 DLFcnError = toString(std::move(Err));
769 return nullptr;
770 }
771
772 // Sign callable symbols as functions, to match dyld.
773 if ((Result.second & MachOExecutorSymbolFlags::Callable) ==
774 MachOExecutorSymbolFlags::Callable)
775 return reinterpret_cast<void *>(Result.first.toPtr<void(void)>());
776 return Result.first.toPtr<void *>();
777 }
778
registerAtExit(void (* F)(void *),void * Arg,void * DSOHandle)779 int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg,
780 void *DSOHandle) {
781 // FIXME: Handle out-of-memory errors, returning -1 if OOM.
782 std::lock_guard<std::mutex> Lock(JDStatesMutex);
783 auto *JDS = getJITDylibStateByHeader(DSOHandle);
784 if (!JDS) {
785 ORC_RT_DEBUG({
786 printdbg("MachOPlatformRuntimeState::registerAtExit called with "
787 "unrecognized dso handle %p\n",
788 DSOHandle);
789 });
790 return -1;
791 }
792 JDS->AtExits.push_back({F, Arg});
793 return 0;
794 }
795
runAtExits(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)796 void MachOPlatformRuntimeState::runAtExits(
797 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
798 auto AtExits = std::move(JDS.AtExits);
799
800 // Unlock while running atexits, as they may trigger operations that modify
801 // JDStates.
802 JDStatesLock.unlock();
803 while (!AtExits.empty()) {
804 auto &AE = AtExits.back();
805 AE.Func(AE.Arg);
806 AtExits.pop_back();
807 }
808 JDStatesLock.lock();
809 }
810
runAtExits(void * DSOHandle)811 void MachOPlatformRuntimeState::runAtExits(void *DSOHandle) {
812 std::unique_lock<std::mutex> Lock(JDStatesMutex);
813 auto *JDS = getJITDylibStateByHeader(DSOHandle);
814 ORC_RT_DEBUG({
815 printdbg("MachOPlatformRuntimeState::runAtExits called on unrecognized "
816 "dso_handle %p\n",
817 DSOHandle);
818 });
819 if (JDS)
820 runAtExits(Lock, *JDS);
821 }
822
823 Expected<std::pair<const char *, size_t>>
getThreadDataSectionFor(const char * ThreadData)824 MachOPlatformRuntimeState::getThreadDataSectionFor(const char *ThreadData) {
825 std::lock_guard<std::mutex> Lock(ThreadDataSectionsMutex);
826 auto I = ThreadDataSections.upper_bound(ThreadData);
827 // Check that we have a valid entry covering this address.
828 if (I == ThreadDataSections.begin())
829 return make_error<StringError>("No thread local data section for key");
830 I = std::prev(I);
831 if (ThreadData >= I->first + I->second)
832 return make_error<StringError>("No thread local data section for key");
833 return *I;
834 }
835
836 MachOPlatformRuntimeState::JITDylibState *
getJITDylibStateByHeader(void * DSOHandle)837 MachOPlatformRuntimeState::getJITDylibStateByHeader(void *DSOHandle) {
838 auto I = JDStates.find(DSOHandle);
839 if (I == JDStates.end()) {
840 I = JDStates.insert(std::make_pair(DSOHandle, JITDylibState())).first;
841 I->second.Header = DSOHandle;
842 }
843 return &I->second;
844 }
845
846 MachOPlatformRuntimeState::JITDylibState *
getJITDylibStateByName(std::string_view Name)847 MachOPlatformRuntimeState::getJITDylibStateByName(std::string_view Name) {
848 // FIXME: Avoid creating string once we have C++20.
849 auto I = JDNameToHeader.find(std::string(Name.data(), Name.size()));
850 if (I != JDNameToHeader.end())
851 return getJITDylibStateByHeader(I->second);
852 return nullptr;
853 }
854
requestPushSymbols(JITDylibState & JDS,span<std::pair<std::string_view,bool>> Symbols)855 Error MachOPlatformRuntimeState::requestPushSymbols(
856 JITDylibState &JDS, span<std::pair<std::string_view, bool>> Symbols) {
857 Error OpErr = Error::success();
858 if (auto Err = WrapperFunction<SPSError(
859 SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>)>::
860 call(JITDispatch(&__orc_rt_macho_push_symbols_tag), OpErr,
861 ExecutorAddr::fromPtr(JDS.Header), Symbols)) {
862 cantFail(std::move(OpErr));
863 return std::move(Err);
864 }
865 return OpErr;
866 }
867
lookupSymbols(JITDylibState & JDS,std::unique_lock<std::mutex> & JDStatesLock,span<std::pair<ExecutorAddr,MachOExecutorSymbolFlags>> Result,span<std::pair<std::string_view,bool>> Symbols)868 Error MachOPlatformRuntimeState::lookupSymbols(
869 JITDylibState &JDS, std::unique_lock<std::mutex> &JDStatesLock,
870 span<std::pair<ExecutorAddr, MachOExecutorSymbolFlags>> Result,
871 span<std::pair<std::string_view, bool>> Symbols) {
872 assert(JDStatesLock.owns_lock() &&
873 "JDStatesLock should be locked at call-site");
874 assert(Result.size() == Symbols.size() &&
875 "Results and Symbols span sizes should match");
876
877 // Make an initial pass over the local symbol table.
878 std::vector<size_t> MissingSymbolIndexes;
879 for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) {
880 auto I = JDS.SymbolTable.find(Symbols[Idx].first);
881 if (I != JDS.SymbolTable.end())
882 Result[Idx] = I->second;
883 else
884 MissingSymbolIndexes.push_back(Idx);
885 }
886
887 // If everything has been resolved already then bail out early.
888 if (MissingSymbolIndexes.empty())
889 return Error::success();
890
891 // Otherwise call back to the controller to try to request that the symbol
892 // be materialized.
893 std::vector<std::pair<std::string_view, bool>> MissingSymbols;
894 MissingSymbols.reserve(MissingSymbolIndexes.size());
895 ORC_RT_DEBUG({
896 printdbg("requesting push of %i missing symbols...\n",
897 MissingSymbolIndexes.size());
898 });
899 for (auto MissingIdx : MissingSymbolIndexes)
900 MissingSymbols.push_back(Symbols[MissingIdx]);
901
902 JDStatesLock.unlock();
903 if (auto Err = requestPushSymbols(
904 JDS, {MissingSymbols.data(), MissingSymbols.size()}))
905 return Err;
906 JDStatesLock.lock();
907
908 // Try to resolve the previously missing symbols locally.
909 std::vector<size_t> MissingRequiredSymbols;
910 for (auto MissingIdx : MissingSymbolIndexes) {
911 auto I = JDS.SymbolTable.find(Symbols[MissingIdx].first);
912 if (I != JDS.SymbolTable.end())
913 Result[MissingIdx] = I->second;
914 else {
915 if (Symbols[MissingIdx].second)
916 MissingRequiredSymbols.push_back(MissingIdx);
917 else
918 Result[MissingIdx] = {ExecutorAddr(), {}};
919 }
920 }
921
922 // Error out if any missing symbols could not be resolved.
923 if (!MissingRequiredSymbols.empty()) {
924 std::ostringstream ErrStream;
925 ErrStream << "Lookup could not find required symbols: [ ";
926 for (auto MissingIdx : MissingRequiredSymbols)
927 ErrStream << "\"" << Symbols[MissingIdx].first << "\" ";
928 ErrStream << "]";
929 return make_error<StringError>(ErrStream.str());
930 }
931
932 return Error::success();
933 }
934
935 // eh-frame registration functions.
936 // We expect these to be available for all processes.
937 extern "C" void __register_frame(const void *);
938 extern "C" void __deregister_frame(const void *);
939
940 template <typename HandleFDEFn>
walkEHFrameSection(span<const char> EHFrameSection,HandleFDEFn HandleFDE)941 void walkEHFrameSection(span<const char> EHFrameSection,
942 HandleFDEFn HandleFDE) {
943 const char *CurCFIRecord = EHFrameSection.data();
944 uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
945
946 while (CurCFIRecord != EHFrameSection.end() && Size != 0) {
947 const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
948 if (Size == 0xffffffff)
949 Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
950 else
951 Size += 4;
952 uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField);
953
954 if (Offset != 0)
955 HandleFDE(CurCFIRecord);
956
957 CurCFIRecord += Size;
958 Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
959 }
960 }
961
lookupUnwindSections(void * Addr,unw_dynamic_unwind_sections & Info)962 bool MachOPlatformRuntimeState::lookupUnwindSections(
963 void *Addr, unw_dynamic_unwind_sections &Info) {
964 ORC_RT_DEBUG(
965 { printdbg("Tried to lookup unwind-info via new lookup call.\n"); });
966 std::lock_guard<std::mutex> Lock(JDStatesMutex);
967 for (auto &KV : JDStates) {
968 auto &JD = KV.second;
969 auto I = JD.UnwindSections.find(reinterpret_cast<char *>(Addr));
970 if (I != JD.UnwindSections.end()) {
971 Info.dso_base = reinterpret_cast<uintptr_t>(JD.Header);
972 Info.dwarf_section =
973 reinterpret_cast<uintptr_t>(I->second.DwarfSection.data());
974 Info.dwarf_section_length = I->second.DwarfSection.size();
975 Info.compact_unwind_section =
976 reinterpret_cast<uintptr_t>(I->second.CompactUnwindSection.data());
977 Info.compact_unwind_section_length =
978 I->second.CompactUnwindSection.size();
979 return true;
980 }
981 }
982 return false;
983 }
984
findDynamicUnwindSections(uintptr_t addr,unw_dynamic_unwind_sections * info)985 int MachOPlatformRuntimeState::findDynamicUnwindSections(
986 uintptr_t addr, unw_dynamic_unwind_sections *info) {
987 if (!info)
988 return 0;
989 return MachOPlatformRuntimeState::get().lookupUnwindSections((void *)addr,
990 *info);
991 }
992
registerEHFrames(span<const char> EHFrameSection)993 Error MachOPlatformRuntimeState::registerEHFrames(
994 span<const char> EHFrameSection) {
995 walkEHFrameSection(EHFrameSection, __register_frame);
996 return Error::success();
997 }
998
deregisterEHFrames(span<const char> EHFrameSection)999 Error MachOPlatformRuntimeState::deregisterEHFrames(
1000 span<const char> EHFrameSection) {
1001 walkEHFrameSection(EHFrameSection, __deregister_frame);
1002 return Error::success();
1003 }
1004
registerObjCRegistrationObjects(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1005 Error MachOPlatformRuntimeState::registerObjCRegistrationObjects(
1006 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1007 ORC_RT_DEBUG(printdbg("Registering Objective-C / Swift metadata.\n"));
1008
1009 std::vector<char *> RegObjBases;
1010 JDS.ObjCRuntimeRegistrationObjects.processNewSections(
1011 [&](span<char> RegObj) { RegObjBases.push_back(RegObj.data()); });
1012
1013 if (RegObjBases.empty())
1014 return Error::success();
1015
1016 if (!_objc_map_images || !_objc_load_image)
1017 return make_error<StringError>(
1018 "Could not register Objective-C / Swift metadata: _objc_map_images / "
1019 "_objc_load_image not found");
1020
1021 // Release the lock while calling out to libobjc in case +load methods cause
1022 // reentering the orc runtime.
1023 JDStatesLock.unlock();
1024 std::vector<char *> Paths;
1025 Paths.resize(RegObjBases.size());
1026 _objc_map_images(RegObjBases.size(), Paths.data(),
1027 reinterpret_cast<mach_header **>(RegObjBases.data()));
1028
1029 for (void *RegObjBase : RegObjBases)
1030 _objc_load_image(nullptr, reinterpret_cast<mach_header *>(RegObjBase));
1031 JDStatesLock.lock();
1032
1033 return Error::success();
1034 }
1035
runModInits(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1036 Error MachOPlatformRuntimeState::runModInits(
1037 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1038 std::vector<span<void (*)()>> InitSections;
1039 InitSections.reserve(JDS.ModInitsSections.numNewSections());
1040
1041 // Copy initializer sections: If the JITDylib is unsealed then the
1042 // initializers could reach back into the JIT and cause more initializers to
1043 // be added.
1044 // FIXME: Skip unlock and run in-place on sealed JITDylibs?
1045 JDS.ModInitsSections.processNewSections(
1046 [&](span<void (*)()> Inits) { InitSections.push_back(Inits); });
1047
1048 JDStatesLock.unlock();
1049 for (auto InitSec : InitSections)
1050 for (auto *Init : InitSec)
1051 Init();
1052 JDStatesLock.lock();
1053
1054 return Error::success();
1055 }
1056
dlopenImpl(std::string_view Path,int Mode)1057 Expected<void *> MachOPlatformRuntimeState::dlopenImpl(std::string_view Path,
1058 int Mode) {
1059 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1060
1061 // Try to find JITDylib state by name.
1062 auto *JDS = getJITDylibStateByName(Path);
1063
1064 if (!JDS)
1065 return make_error<StringError>("No registered JTIDylib for path " +
1066 std::string(Path.data(), Path.size()));
1067
1068 // If this JITDylib is unsealed, or this is the first dlopen then run
1069 // full dlopen path (update deps, push and run initializers, update ref
1070 // counts on all JITDylibs in the dep tree).
1071 if (!JDS->referenced() || !JDS->Sealed) {
1072 if (auto Err = dlopenFull(Lock, *JDS))
1073 return std::move(Err);
1074 }
1075
1076 // Bump the ref-count on this dylib.
1077 ++JDS->DlRefCount;
1078
1079 // Return the header address.
1080 return JDS->Header;
1081 }
1082
dlopenFull(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1083 Error MachOPlatformRuntimeState::dlopenFull(
1084 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1085 // Call back to the JIT to push the initializers.
1086 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1087 // Unlock so that we can accept the initializer update.
1088 JDStatesLock.unlock();
1089 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1090 SPSExecutorAddr)>::
1091 call(JITDispatch(&__orc_rt_macho_push_initializers_tag), DepInfo,
1092 ExecutorAddr::fromPtr(JDS.Header)))
1093 return Err;
1094 JDStatesLock.lock();
1095
1096 if (!DepInfo)
1097 return DepInfo.takeError();
1098
1099 if (auto Err = dlopenInitialize(JDStatesLock, JDS, *DepInfo))
1100 return Err;
1101
1102 if (!DepInfo->empty()) {
1103 ORC_RT_DEBUG({
1104 printdbg("Unrecognized dep-info key headers in dlopen of %s\n",
1105 JDS.Name.c_str());
1106 });
1107 std::ostringstream ErrStream;
1108 ErrStream << "Encountered unrecognized dep-info key headers "
1109 "while processing dlopen of "
1110 << JDS.Name;
1111 return make_error<StringError>(ErrStream.str());
1112 }
1113
1114 return Error::success();
1115 }
1116
dlopenInitialize(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS,MachOJITDylibDepInfoMap & DepInfo)1117 Error MachOPlatformRuntimeState::dlopenInitialize(
1118 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS,
1119 MachOJITDylibDepInfoMap &DepInfo) {
1120 ORC_RT_DEBUG({
1121 printdbg("MachOPlatformRuntimeState::dlopenInitialize(\"%s\")\n",
1122 JDS.Name.c_str());
1123 });
1124
1125 // If the header is not present in the dep map then assume that we
1126 // already processed it earlier in the dlopenInitialize traversal and
1127 // return.
1128 // TODO: Keep a visited set instead so that we can error out on missing
1129 // entries?
1130 auto I = DepInfo.find(ExecutorAddr::fromPtr(JDS.Header));
1131 if (I == DepInfo.end())
1132 return Error::success();
1133
1134 auto DI = std::move(I->second);
1135 DepInfo.erase(I);
1136
1137 // We don't need to re-initialize sealed JITDylibs that have already been
1138 // initialized. Just check that their dep-map entry is empty as expected.
1139 if (JDS.Sealed) {
1140 if (!DI.DepHeaders.empty()) {
1141 std::ostringstream ErrStream;
1142 ErrStream << "Sealed JITDylib " << JDS.Header
1143 << " already has registered dependencies";
1144 return make_error<StringError>(ErrStream.str());
1145 }
1146 if (JDS.referenced())
1147 return Error::success();
1148 } else
1149 JDS.Sealed = DI.Sealed;
1150
1151 // This is an unsealed or newly sealed JITDylib. Run initializers.
1152 std::vector<JITDylibState *> OldDeps;
1153 std::swap(JDS.Deps, OldDeps);
1154 JDS.Deps.reserve(DI.DepHeaders.size());
1155 for (auto DepHeaderAddr : DI.DepHeaders) {
1156 auto *DepJDS = getJITDylibStateByHeader(DepHeaderAddr.toPtr<void *>());
1157 if (!DepJDS) {
1158 std::ostringstream ErrStream;
1159 ErrStream << "Encountered unrecognized dep header "
1160 << DepHeaderAddr.toPtr<void *>() << " while initializing "
1161 << JDS.Name;
1162 return make_error<StringError>(ErrStream.str());
1163 }
1164 ++DepJDS->LinkedAgainstRefCount;
1165 if (auto Err = dlopenInitialize(JDStatesLock, *DepJDS, DepInfo))
1166 return Err;
1167 }
1168
1169 // Initialize this JITDylib.
1170 if (auto Err = registerObjCRegistrationObjects(JDStatesLock, JDS))
1171 return Err;
1172 if (auto Err = runModInits(JDStatesLock, JDS))
1173 return Err;
1174
1175 // Decrement old deps.
1176 // FIXME: We should probably continue and just report deinitialize errors
1177 // here.
1178 for (auto *DepJDS : OldDeps) {
1179 --DepJDS->LinkedAgainstRefCount;
1180 if (!DepJDS->referenced())
1181 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1182 return Err;
1183 }
1184
1185 return Error::success();
1186 }
1187
dlupdateImpl(void * DSOHandle)1188 Error MachOPlatformRuntimeState::dlupdateImpl(void *DSOHandle) {
1189 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1190
1191 // Try to find JITDylib state by DSOHandle.
1192 auto *JDS = getJITDylibStateByHeader(DSOHandle);
1193
1194 if (!JDS) {
1195 std::ostringstream ErrStream;
1196 ErrStream << "No registered JITDylib for " << DSOHandle;
1197 return make_error<StringError>(ErrStream.str());
1198 }
1199
1200 if (!JDS->referenced())
1201 return make_error<StringError>("dlupdate failed, JITDylib must be open.");
1202
1203 if (!JDS->Sealed) {
1204 if (auto Err = dlupdateFull(Lock, *JDS))
1205 return Err;
1206 }
1207
1208 return Error::success();
1209 }
1210
dlupdateFull(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1211 Error MachOPlatformRuntimeState::dlupdateFull(
1212 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1213 // Call back to the JIT to push the initializers.
1214 Expected<MachOJITDylibDepInfoMap> DepInfo((MachOJITDylibDepInfoMap()));
1215 // Unlock so that we can accept the initializer update.
1216 JDStatesLock.unlock();
1217 if (auto Err = WrapperFunction<SPSExpected<SPSMachOJITDylibDepInfoMap>(
1218 SPSExecutorAddr)>::
1219 call(JITDispatch(&__orc_rt_macho_push_initializers_tag), DepInfo,
1220 ExecutorAddr::fromPtr(JDS.Header)))
1221 return Err;
1222 JDStatesLock.lock();
1223
1224 if (!DepInfo)
1225 return DepInfo.takeError();
1226
1227 if (auto Err = dlupdateInitialize(JDStatesLock, JDS))
1228 return Err;
1229
1230 return Error::success();
1231 }
1232
dlupdateInitialize(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1233 Error MachOPlatformRuntimeState::dlupdateInitialize(
1234 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1235 ORC_RT_DEBUG({
1236 printdbg("MachOPlatformRuntimeState::dlupdateInitialize(\"%s\")\n",
1237 JDS.Name.c_str());
1238 });
1239
1240 // Initialize this JITDylib.
1241 if (auto Err = registerObjCRegistrationObjects(JDStatesLock, JDS))
1242 return Err;
1243 if (auto Err = runModInits(JDStatesLock, JDS))
1244 return Err;
1245
1246 return Error::success();
1247 }
1248
dlcloseImpl(void * DSOHandle)1249 Error MachOPlatformRuntimeState::dlcloseImpl(void *DSOHandle) {
1250 std::unique_lock<std::mutex> Lock(JDStatesMutex);
1251
1252 // Try to find JITDylib state by header.
1253 auto *JDS = getJITDylibStateByHeader(DSOHandle);
1254
1255 if (!JDS) {
1256 std::ostringstream ErrStream;
1257 ErrStream << "No registered JITDylib for " << DSOHandle;
1258 return make_error<StringError>(ErrStream.str());
1259 }
1260
1261 // Bump the ref-count.
1262 --JDS->DlRefCount;
1263
1264 if (!JDS->referenced())
1265 return dlcloseDeinitialize(Lock, *JDS);
1266
1267 return Error::success();
1268 }
1269
dlcloseDeinitialize(std::unique_lock<std::mutex> & JDStatesLock,JITDylibState & JDS)1270 Error MachOPlatformRuntimeState::dlcloseDeinitialize(
1271 std::unique_lock<std::mutex> &JDStatesLock, JITDylibState &JDS) {
1272
1273 ORC_RT_DEBUG({
1274 printdbg("MachOPlatformRuntimeState::dlcloseDeinitialize(\"%s\")\n",
1275 JDS.Name.c_str());
1276 });
1277
1278 runAtExits(JDStatesLock, JDS);
1279
1280 // Reset mod-inits
1281 JDS.ModInitsSections.reset();
1282
1283 // Reset data section contents.
1284 for (auto &KV : JDS.DataSectionContent)
1285 memcpy(KV.first, KV.second.data(), KV.second.size());
1286 for (auto &KV : JDS.ZeroInitRanges)
1287 memset(KV.first, 0, KV.second);
1288
1289 // Deinitialize any dependencies.
1290 for (auto *DepJDS : JDS.Deps) {
1291 --DepJDS->LinkedAgainstRefCount;
1292 if (!DepJDS->referenced())
1293 if (auto Err = dlcloseDeinitialize(JDStatesLock, *DepJDS))
1294 return Err;
1295 }
1296
1297 return Error::success();
1298 }
1299
1300 class MachOPlatformRuntimeTLVManager {
1301 public:
1302 void *getInstance(const char *ThreadData);
1303
1304 private:
1305 std::unordered_map<const char *, char *> Instances;
1306 std::unordered_map<const char *, std::unique_ptr<char[]>> AllocatedSections;
1307 };
1308
getInstance(const char * ThreadData)1309 void *MachOPlatformRuntimeTLVManager::getInstance(const char *ThreadData) {
1310 auto I = Instances.find(ThreadData);
1311 if (I != Instances.end())
1312 return I->second;
1313
1314 auto TDS =
1315 MachOPlatformRuntimeState::get().getThreadDataSectionFor(ThreadData);
1316 if (!TDS) {
1317 __orc_rt_log_error(toString(TDS.takeError()).c_str());
1318 return nullptr;
1319 }
1320
1321 auto &Allocated = AllocatedSections[TDS->first];
1322 if (!Allocated) {
1323 Allocated = std::make_unique<char[]>(TDS->second);
1324 memcpy(Allocated.get(), TDS->first, TDS->second);
1325 }
1326
1327 size_t ThreadDataDelta = ThreadData - TDS->first;
1328 assert(ThreadDataDelta <= TDS->second && "ThreadData outside section bounds");
1329
1330 char *Instance = Allocated.get() + ThreadDataDelta;
1331 Instances[ThreadData] = Instance;
1332 return Instance;
1333 }
1334
destroyMachOTLVMgr(void * MachOTLVMgr)1335 void destroyMachOTLVMgr(void *MachOTLVMgr) {
1336 delete static_cast<MachOPlatformRuntimeTLVManager *>(MachOTLVMgr);
1337 }
1338
runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs)1339 Error runWrapperFunctionCalls(std::vector<WrapperFunctionCall> WFCs) {
1340 for (auto &WFC : WFCs)
1341 if (auto Err = WFC.runWithSPSRet<void>())
1342 return Err;
1343 return Error::success();
1344 }
1345
1346 } // end anonymous namespace
1347
1348 //------------------------------------------------------------------------------
1349 // JIT entry points
1350 //------------------------------------------------------------------------------
1351
1352 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_platform_bootstrap(char * ArgData,size_t ArgSize)1353 __orc_rt_macho_platform_bootstrap(char *ArgData, size_t ArgSize) {
1354 return WrapperFunction<SPSError()>::handle(
1355 ArgData, ArgSize,
1356 []() { return MachOPlatformRuntimeState::create(); })
1357 .release();
1358 }
1359
1360 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_platform_shutdown(char * ArgData,size_t ArgSize)1361 __orc_rt_macho_platform_shutdown(char *ArgData, size_t ArgSize) {
1362 return WrapperFunction<SPSError()>::handle(
1363 ArgData, ArgSize,
1364 []() { return MachOPlatformRuntimeState::destroy(); })
1365 .release();
1366 }
1367
1368 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_register_jitdylib(char * ArgData,size_t ArgSize)1369 __orc_rt_macho_register_jitdylib(char *ArgData, size_t ArgSize) {
1370 return WrapperFunction<SPSError(SPSString, SPSExecutorAddr)>::handle(
1371 ArgData, ArgSize,
1372 [](std::string &Name, ExecutorAddr HeaderAddr) {
1373 return MachOPlatformRuntimeState::get().registerJITDylib(
1374 std::move(Name), HeaderAddr.toPtr<void *>());
1375 })
1376 .release();
1377 }
1378
1379 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_deregister_jitdylib(char * ArgData,size_t ArgSize)1380 __orc_rt_macho_deregister_jitdylib(char *ArgData, size_t ArgSize) {
1381 return WrapperFunction<SPSError(SPSExecutorAddr)>::handle(
1382 ArgData, ArgSize,
1383 [](ExecutorAddr HeaderAddr) {
1384 return MachOPlatformRuntimeState::get().deregisterJITDylib(
1385 HeaderAddr.toPtr<void *>());
1386 })
1387 .release();
1388 }
1389
1390 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_register_object_platform_sections(char * ArgData,size_t ArgSize)1391 __orc_rt_macho_register_object_platform_sections(char *ArgData,
1392 size_t ArgSize) {
1393 return WrapperFunction<SPSError(SPSExecutorAddr,
1394 SPSOptional<SPSUnwindSectionInfo>,
1395 SPSMachOObjectPlatformSectionsMap)>::
1396 handle(ArgData, ArgSize,
1397 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1398 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1399 &Secs) {
1400 return MachOPlatformRuntimeState::get()
1401 .registerObjectPlatformSections(HeaderAddr, std::move(USI),
1402 std::move(Secs));
1403 })
1404 .release();
1405 }
1406
1407 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_register_object_symbol_table(char * ArgData,size_t ArgSize)1408 __orc_rt_macho_register_object_symbol_table(char *ArgData, size_t ArgSize) {
1409 using SymtabContainer = std::vector<
1410 std::tuple<ExecutorAddr, ExecutorAddr,
1411 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1412 return WrapperFunction<SPSError(
1413 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1414 SPSMachOExecutorSymbolFlags>>)>::
1415 handle(ArgData, ArgSize,
1416 [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1417 return MachOPlatformRuntimeState::get()
1418 .registerObjectSymbolTable(HeaderAddr, Symbols);
1419 })
1420 .release();
1421 }
1422
1423 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_deregister_object_symbol_table(char * ArgData,size_t ArgSize)1424 __orc_rt_macho_deregister_object_symbol_table(char *ArgData, size_t ArgSize) {
1425 using SymtabContainer = std::vector<
1426 std::tuple<ExecutorAddr, ExecutorAddr,
1427 MachOPlatformRuntimeState::MachOExecutorSymbolFlags>>;
1428 return WrapperFunction<SPSError(
1429 SPSExecutorAddr, SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr,
1430 SPSMachOExecutorSymbolFlags>>)>::
1431 handle(ArgData, ArgSize,
1432 [](ExecutorAddr HeaderAddr, SymtabContainer &Symbols) {
1433 return MachOPlatformRuntimeState::get()
1434 .deregisterObjectSymbolTable(HeaderAddr, Symbols);
1435 })
1436 .release();
1437 }
1438
1439 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_deregister_object_platform_sections(char * ArgData,size_t ArgSize)1440 __orc_rt_macho_deregister_object_platform_sections(char *ArgData,
1441 size_t ArgSize) {
1442 return WrapperFunction<SPSError(SPSExecutorAddr,
1443 SPSOptional<SPSUnwindSectionInfo>,
1444 SPSMachOObjectPlatformSectionsMap)>::
1445 handle(ArgData, ArgSize,
1446 [](ExecutorAddr HeaderAddr, std::optional<UnwindSectionInfo> USI,
1447 std::vector<std::pair<std::string_view, ExecutorAddrRange>>
1448 &Secs) {
1449 return MachOPlatformRuntimeState::get()
1450 .deregisterObjectPlatformSections(HeaderAddr, std::move(USI),
1451 std::move(Secs));
1452 })
1453 .release();
1454 }
1455
1456 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_run_wrapper_function_calls(char * ArgData,size_t ArgSize)1457 __orc_rt_macho_run_wrapper_function_calls(char *ArgData, size_t ArgSize) {
1458 return WrapperFunction<SPSError(SPSSequence<SPSWrapperFunctionCall>)>::handle(
1459 ArgData, ArgSize, runWrapperFunctionCalls)
1460 .release();
1461 }
1462
1463 //------------------------------------------------------------------------------
1464 // TLV support
1465 //------------------------------------------------------------------------------
1466
__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor * D)1467 ORC_RT_INTERFACE void *__orc_rt_macho_tlv_get_addr_impl(TLVDescriptor *D) {
1468 auto *TLVMgr = static_cast<MachOPlatformRuntimeTLVManager *>(
1469 pthread_getspecific(D->Key));
1470 if (!TLVMgr) {
1471 TLVMgr = new MachOPlatformRuntimeTLVManager();
1472 if (pthread_setspecific(D->Key, TLVMgr)) {
1473 __orc_rt_log_error("Call to pthread_setspecific failed");
1474 return nullptr;
1475 }
1476 }
1477
1478 return TLVMgr->getInstance(
1479 reinterpret_cast<char *>(static_cast<uintptr_t>(D->DataAddress)));
1480 }
1481
1482 ORC_RT_INTERFACE orc_rt_WrapperFunctionResult
__orc_rt_macho_create_pthread_key(char * ArgData,size_t ArgSize)1483 __orc_rt_macho_create_pthread_key(char *ArgData, size_t ArgSize) {
1484 return WrapperFunction<SPSExpected<uint64_t>(void)>::handle(
1485 ArgData, ArgSize,
1486 []() -> Expected<uint64_t> {
1487 pthread_key_t Key;
1488 if (int Err = pthread_key_create(&Key, destroyMachOTLVMgr)) {
1489 __orc_rt_log_error("Call to pthread_key_create failed");
1490 return make_error<StringError>(strerror(Err));
1491 }
1492 return static_cast<uint64_t>(Key);
1493 })
1494 .release();
1495 }
1496
1497 //------------------------------------------------------------------------------
1498 // cxa_atexit support
1499 //------------------------------------------------------------------------------
1500
__orc_rt_macho_cxa_atexit(void (* func)(void *),void * arg,void * dso_handle)1501 int __orc_rt_macho_cxa_atexit(void (*func)(void *), void *arg,
1502 void *dso_handle) {
1503 return MachOPlatformRuntimeState::get().registerAtExit(func, arg, dso_handle);
1504 }
1505
__orc_rt_macho_cxa_finalize(void * dso_handle)1506 void __orc_rt_macho_cxa_finalize(void *dso_handle) {
1507 MachOPlatformRuntimeState::get().runAtExits(dso_handle);
1508 }
1509
1510 //------------------------------------------------------------------------------
1511 // JIT'd dlfcn alternatives.
1512 //------------------------------------------------------------------------------
1513
__orc_rt_macho_jit_dlerror()1514 const char *__orc_rt_macho_jit_dlerror() {
1515 return MachOPlatformRuntimeState::get().dlerror();
1516 }
1517
__orc_rt_macho_jit_dlopen(const char * path,int mode)1518 void *__orc_rt_macho_jit_dlopen(const char *path, int mode) {
1519 return MachOPlatformRuntimeState::get().dlopen(path, mode);
1520 }
1521
__orc_rt_macho_jit_dlupdate(void * dso_handle)1522 int __orc_rt_macho_jit_dlupdate(void *dso_handle) {
1523 return MachOPlatformRuntimeState::get().dlupdate(dso_handle);
1524 }
1525
__orc_rt_macho_jit_dlclose(void * dso_handle)1526 int __orc_rt_macho_jit_dlclose(void *dso_handle) {
1527 return MachOPlatformRuntimeState::get().dlclose(dso_handle);
1528 }
1529
__orc_rt_macho_jit_dlsym(void * dso_handle,const char * symbol)1530 void *__orc_rt_macho_jit_dlsym(void *dso_handle, const char *symbol) {
1531 return MachOPlatformRuntimeState::get().dlsym(dso_handle, symbol);
1532 }
1533
1534 //------------------------------------------------------------------------------
1535 // MachO Run Program
1536 //------------------------------------------------------------------------------
1537
__orc_rt_macho_run_program(const char * JITDylibName,const char * EntrySymbolName,int argc,char * argv[])1538 ORC_RT_INTERFACE int64_t __orc_rt_macho_run_program(const char *JITDylibName,
1539 const char *EntrySymbolName,
1540 int argc, char *argv[]) {
1541 using MainTy = int (*)(int, char *[]);
1542
1543 void *H =
1544 __orc_rt_macho_jit_dlopen(JITDylibName, orc_rt::macho::ORC_RT_RTLD_LAZY);
1545 if (!H) {
1546 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1547 return -1;
1548 }
1549
1550 auto *Main =
1551 reinterpret_cast<MainTy>(__orc_rt_macho_jit_dlsym(H, EntrySymbolName));
1552
1553 if (!Main) {
1554 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1555 return -1;
1556 }
1557
1558 int Result = Main(argc, argv);
1559
1560 if (__orc_rt_macho_jit_dlclose(H) == -1)
1561 __orc_rt_log_error(__orc_rt_macho_jit_dlerror());
1562
1563 return Result;
1564 }
1565