1 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10
11 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
12 #include "llvm/ExecutionEngine/Orc/COFF.h"
13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
15 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
16 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
17
18 #include "llvm/Object/COFF.h"
19
20 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
21
22 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
23
24 #define DEBUG_TYPE "orc"
25
26 using namespace llvm;
27 using namespace llvm::orc;
28 using namespace llvm::orc::shared;
29
30 namespace llvm {
31 namespace orc {
32 namespace shared {
33
34 using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
35 using SPSCOFFJITDylibDepInfoMap =
36 SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
37 using SPSCOFFObjectSectionsMap =
38 SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
39 using SPSCOFFRegisterObjectSectionsArgs =
40 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
41 using SPSCOFFDeregisterObjectSectionsArgs =
42 SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
43
44 } // namespace shared
45 } // namespace orc
46 } // namespace llvm
47 namespace {
48
49 class COFFHeaderMaterializationUnit : public MaterializationUnit {
50 public:
COFFHeaderMaterializationUnit(COFFPlatform & CP,const SymbolStringPtr & HeaderStartSymbol)51 COFFHeaderMaterializationUnit(COFFPlatform &CP,
52 const SymbolStringPtr &HeaderStartSymbol)
53 : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),
54 CP(CP) {}
55
getName() const56 StringRef getName() const override { return "COFFHeaderMU"; }
57
materialize(std::unique_ptr<MaterializationResponsibility> R)58 void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
59 auto G = std::make_unique<jitlink::LinkGraph>(
60 "<COFFHeaderMU>", CP.getExecutionSession().getSymbolStringPool(),
61 CP.getExecutionSession().getTargetTriple(), SubtargetFeatures(),
62 jitlink::getGenericEdgeKindName);
63 auto &HeaderSection = G->createSection("__header", MemProt::Read);
64 auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
65
66 // Init symbol is __ImageBase symbol.
67 auto &ImageBaseSymbol = G->addDefinedSymbol(
68 HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),
69 jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
70
71 addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);
72
73 CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
74 }
75
discard(const JITDylib & JD,const SymbolStringPtr & Sym)76 void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
77
78 private:
79 struct HeaderSymbol {
80 const char *Name;
81 uint64_t Offset;
82 };
83
84 struct NTHeader {
85 support::ulittle32_t PEMagic;
86 object::coff_file_header FileHeader;
87 struct PEHeader {
88 object::pe32plus_header Header;
89 object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
90 } OptionalHeader;
91 };
92
93 struct HeaderBlockContent {
94 object::dos_header DOSHeader;
95 COFFHeaderMaterializationUnit::NTHeader NTHeader;
96 };
97
createHeaderBlock(jitlink::LinkGraph & G,jitlink::Section & HeaderSection)98 static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
99 jitlink::Section &HeaderSection) {
100 HeaderBlockContent Hdr = {};
101
102 // Set up magic
103 Hdr.DOSHeader.Magic[0] = 'M';
104 Hdr.DOSHeader.Magic[1] = 'Z';
105 Hdr.DOSHeader.AddressOfNewExeHeader =
106 offsetof(HeaderBlockContent, NTHeader);
107 uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
108 Hdr.NTHeader.PEMagic = PEMagic;
109 Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
110
111 switch (G.getTargetTriple().getArch()) {
112 case Triple::x86_64:
113 Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
114 break;
115 default:
116 llvm_unreachable("Unrecognized architecture");
117 }
118
119 auto HeaderContent = G.allocateContent(
120 ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
121
122 return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
123 0);
124 }
125
addImageBaseRelocationEdge(jitlink::Block & B,jitlink::Symbol & ImageBase)126 static void addImageBaseRelocationEdge(jitlink::Block &B,
127 jitlink::Symbol &ImageBase) {
128 auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
129 offsetof(NTHeader, OptionalHeader) +
130 offsetof(object::pe32plus_header, ImageBase);
131 B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);
132 }
133
134 static MaterializationUnit::Interface
createHeaderInterface(COFFPlatform & MOP,const SymbolStringPtr & HeaderStartSymbol)135 createHeaderInterface(COFFPlatform &MOP,
136 const SymbolStringPtr &HeaderStartSymbol) {
137 SymbolFlagsMap HeaderSymbolFlags;
138
139 HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
140
141 return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
142 HeaderStartSymbol);
143 }
144
145 COFFPlatform &CP;
146 };
147
148 } // end anonymous namespace
149
150 namespace llvm {
151 namespace orc {
152
153 Expected<std::unique_ptr<COFFPlatform>>
Create(ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,LoadDynamicLibrary LoadDynLibrary,bool StaticVCRuntime,const char * VCRuntimePath,std::optional<SymbolAliasMap> RuntimeAliases)154 COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
155 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
156 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
157 const char *VCRuntimePath,
158 std::optional<SymbolAliasMap> RuntimeAliases) {
159
160 auto &ES = ObjLinkingLayer.getExecutionSession();
161
162 // If the target is not supported then bail out immediately.
163 if (!supportedTarget(ES.getTargetTriple()))
164 return make_error<StringError>("Unsupported COFFPlatform triple: " +
165 ES.getTargetTriple().str(),
166 inconvertibleErrorCode());
167
168 auto &EPC = ES.getExecutorProcessControl();
169
170 auto GeneratorArchive =
171 object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef());
172 if (!GeneratorArchive)
173 return GeneratorArchive.takeError();
174
175 std::set<std::string> DylibsToPreload;
176 auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
177 ObjLinkingLayer, nullptr, std::move(*GeneratorArchive),
178 COFFImportFileScanner(DylibsToPreload));
179 if (!OrcRuntimeArchiveGenerator)
180 return OrcRuntimeArchiveGenerator.takeError();
181
182 // We need a second instance of the archive (for now) for the Platform. We
183 // can `cantFail` this call, since if it were going to fail it would have
184 // failed above.
185 auto RuntimeArchive = cantFail(
186 object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()));
187
188 // Create default aliases if the caller didn't supply any.
189 if (!RuntimeAliases)
190 RuntimeAliases = standardPlatformAliases(ES);
191
192 // Define the aliases.
193 if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
194 return std::move(Err);
195
196 auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");
197
198 // Add JIT-dispatch function support symbols.
199 if (auto Err = HostFuncJD.define(
200 absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
201 {EPC.getJITDispatchInfo().JITDispatchFunction,
202 JITSymbolFlags::Exported}},
203 {ES.intern("__orc_rt_jit_dispatch_ctx"),
204 {EPC.getJITDispatchInfo().JITDispatchContext,
205 JITSymbolFlags::Exported}}})))
206 return std::move(Err);
207
208 PlatformJD.addToLinkOrder(HostFuncJD);
209
210 // Create the instance.
211 Error Err = Error::success();
212 auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
213 ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
214 std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
215 std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
216 VCRuntimePath, Err));
217 if (Err)
218 return std::move(Err);
219 return std::move(P);
220 }
221
222 Expected<std::unique_ptr<COFFPlatform>>
Create(ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,const char * OrcRuntimePath,LoadDynamicLibrary LoadDynLibrary,bool StaticVCRuntime,const char * VCRuntimePath,std::optional<SymbolAliasMap> RuntimeAliases)223 COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
224 const char *OrcRuntimePath,
225 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
226 const char *VCRuntimePath,
227 std::optional<SymbolAliasMap> RuntimeAliases) {
228
229 auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
230 if (!ArchiveBuffer)
231 return createFileError(OrcRuntimePath, ArchiveBuffer.getError());
232
233 return Create(ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer),
234 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
235 std::move(RuntimeAliases));
236 }
237
getPerJDObjectFile()238 Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
239 auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
240 if (!PerJDObj)
241 return PerJDObj.takeError();
242
243 if (!*PerJDObj)
244 return make_error<StringError>("Could not find per jd object file",
245 inconvertibleErrorCode());
246
247 auto Buffer = (*PerJDObj)->getAsBinary();
248 if (!Buffer)
249 return Buffer.takeError();
250
251 return (*Buffer)->getMemoryBufferRef();
252 }
253
addAliases(ExecutionSession & ES,SymbolAliasMap & Aliases,ArrayRef<std::pair<const char *,const char * >> AL)254 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
255 ArrayRef<std::pair<const char *, const char *>> AL) {
256 for (auto &KV : AL) {
257 auto AliasName = ES.intern(KV.first);
258 assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
259 Aliases[std::move(AliasName)] = {ES.intern(KV.second),
260 JITSymbolFlags::Exported};
261 }
262 }
263
setupJITDylib(JITDylib & JD)264 Error COFFPlatform::setupJITDylib(JITDylib &JD) {
265 if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
266 *this, COFFHeaderStartSymbol)))
267 return Err;
268
269 if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
270 return Err;
271
272 // Define the CXX aliases.
273 SymbolAliasMap CXXAliases;
274 addAliases(ES, CXXAliases, requiredCXXAliases());
275 if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
276 return Err;
277
278 auto PerJDObj = getPerJDObjectFile();
279 if (!PerJDObj)
280 return PerJDObj.takeError();
281
282 auto I = getObjectFileInterface(ES, *PerJDObj);
283 if (!I)
284 return I.takeError();
285
286 if (auto Err = ObjLinkingLayer.add(
287 JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
288 return Err;
289
290 if (!Bootstrapping) {
291 auto ImportedLibs = StaticVCRuntime
292 ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
293 : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
294 if (!ImportedLibs)
295 return ImportedLibs.takeError();
296 for (auto &Lib : *ImportedLibs)
297 if (auto Err = LoadDynLibrary(JD, Lib))
298 return Err;
299 if (StaticVCRuntime)
300 if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
301 return Err;
302 }
303
304 JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
305 return Error::success();
306 }
307
teardownJITDylib(JITDylib & JD)308 Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
309 std::lock_guard<std::mutex> Lock(PlatformMutex);
310 auto I = JITDylibToHeaderAddr.find(&JD);
311 if (I != JITDylibToHeaderAddr.end()) {
312 assert(HeaderAddrToJITDylib.count(I->second) &&
313 "HeaderAddrToJITDylib missing entry");
314 HeaderAddrToJITDylib.erase(I->second);
315 JITDylibToHeaderAddr.erase(I);
316 }
317 return Error::success();
318 }
319
notifyAdding(ResourceTracker & RT,const MaterializationUnit & MU)320 Error COFFPlatform::notifyAdding(ResourceTracker &RT,
321 const MaterializationUnit &MU) {
322 auto &JD = RT.getJITDylib();
323 const auto &InitSym = MU.getInitializerSymbol();
324 if (!InitSym)
325 return Error::success();
326
327 RegisteredInitSymbols[&JD].add(InitSym,
328 SymbolLookupFlags::WeaklyReferencedSymbol);
329
330 LLVM_DEBUG({
331 dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
332 << MU.getName() << "\n";
333 });
334 return Error::success();
335 }
336
notifyRemoving(ResourceTracker & RT)337 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
338 llvm_unreachable("Not supported yet");
339 }
340
standardPlatformAliases(ExecutionSession & ES)341 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
342 SymbolAliasMap Aliases;
343 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
344 return Aliases;
345 }
346
347 ArrayRef<std::pair<const char *, const char *>>
requiredCXXAliases()348 COFFPlatform::requiredCXXAliases() {
349 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
350 {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
351 {"_onexit", "__orc_rt_coff_onexit_per_jd"},
352 {"atexit", "__orc_rt_coff_atexit_per_jd"}};
353
354 return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
355 }
356
357 ArrayRef<std::pair<const char *, const char *>>
standardRuntimeUtilityAliases()358 COFFPlatform::standardRuntimeUtilityAliases() {
359 static const std::pair<const char *, const char *>
360 StandardRuntimeUtilityAliases[] = {
361 {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
362 {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
363 {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
364 {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
365 {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
366 {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
367
368 return ArrayRef<std::pair<const char *, const char *>>(
369 StandardRuntimeUtilityAliases);
370 }
371
supportedTarget(const Triple & TT)372 bool COFFPlatform::supportedTarget(const Triple &TT) {
373 switch (TT.getArch()) {
374 case Triple::x86_64:
375 return true;
376 default:
377 return false;
378 }
379 }
380
COFFPlatform(ObjectLinkingLayer & ObjLinkingLayer,JITDylib & PlatformJD,std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,std::set<std::string> DylibsToPreload,std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,std::unique_ptr<object::Archive> OrcRuntimeArchive,LoadDynamicLibrary LoadDynLibrary,bool StaticVCRuntime,const char * VCRuntimePath,Error & Err)381 COFFPlatform::COFFPlatform(
382 ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
383 std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
384 std::set<std::string> DylibsToPreload,
385 std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
386 std::unique_ptr<object::Archive> OrcRuntimeArchive,
387 LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
388 const char *VCRuntimePath, Error &Err)
389 : ES(ObjLinkingLayer.getExecutionSession()),
390 ObjLinkingLayer(ObjLinkingLayer),
391 LoadDynLibrary(std::move(LoadDynLibrary)),
392 OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
393 OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
394 StaticVCRuntime(StaticVCRuntime),
395 COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
396 ErrorAsOutParameter _(Err);
397
398 Bootstrapping.store(true);
399 ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
400
401 // Load vc runtime
402 auto VCRT =
403 COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
404 if (!VCRT) {
405 Err = VCRT.takeError();
406 return;
407 }
408 VCRuntimeBootstrap = std::move(*VCRT);
409
410 auto ImportedLibs =
411 StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
412 : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
413 if (!ImportedLibs) {
414 Err = ImportedLibs.takeError();
415 return;
416 }
417
418 for (auto &Lib : *ImportedLibs)
419 DylibsToPreload.insert(Lib);
420
421 PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
422
423 // PlatformJD hasn't been set up by the platform yet (since we're creating
424 // the platform now), so set it up.
425 if (auto E2 = setupJITDylib(PlatformJD)) {
426 Err = std::move(E2);
427 return;
428 }
429
430 for (auto& Lib : DylibsToPreload)
431 if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
432 Err = std::move(E2);
433 return;
434 }
435
436 if (StaticVCRuntime)
437 if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {
438 Err = std::move(E2);
439 return;
440 }
441
442 // Associate wrapper function tags with JIT-side function implementations.
443 if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
444 Err = std::move(E2);
445 return;
446 }
447
448 // Lookup addresses of runtime functions callable by the platform,
449 // call the platform bootstrap function to initialize the platform-state
450 // object in the executor.
451 if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
452 Err = std::move(E2);
453 return;
454 }
455
456 Bootstrapping.store(false);
457 JDBootstrapStates.clear();
458 }
459
460 Expected<COFFPlatform::JITDylibDepMap>
buildJDDepMap(JITDylib & JD)461 COFFPlatform::buildJDDepMap(JITDylib &JD) {
462 return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {
463 JITDylibDepMap JDDepMap;
464
465 SmallVector<JITDylib *, 16> Worklist({&JD});
466 while (!Worklist.empty()) {
467 auto CurJD = Worklist.back();
468 Worklist.pop_back();
469
470 auto &DM = JDDepMap[CurJD];
471 CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
472 DM.reserve(O.size());
473 for (auto &KV : O) {
474 if (KV.first == CurJD)
475 continue;
476 {
477 // Bare jitdylibs not known to the platform
478 std::lock_guard<std::mutex> Lock(PlatformMutex);
479 if (!JITDylibToHeaderAddr.count(KV.first)) {
480 LLVM_DEBUG({
481 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
482 "LinkOrder: "
483 << CurJD->getName() << "\n";
484 });
485 continue;
486 }
487 }
488 DM.push_back(KV.first);
489 // Push unvisited entry.
490 if (JDDepMap.try_emplace(KV.first).second)
491 Worklist.push_back(KV.first);
492 }
493 });
494 }
495 return std::move(JDDepMap);
496 });
497 }
498
pushInitializersLoop(PushInitializersSendResultFn SendResult,JITDylibSP JD,JITDylibDepMap & JDDepMap)499 void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
500 JITDylibSP JD,
501 JITDylibDepMap &JDDepMap) {
502 SmallVector<JITDylib *, 16> Worklist({JD.get()});
503 DenseSet<JITDylib *> Visited({JD.get()});
504 DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
505 ES.runSessionLocked([&]() {
506 while (!Worklist.empty()) {
507 auto CurJD = Worklist.back();
508 Worklist.pop_back();
509
510 auto RISItr = RegisteredInitSymbols.find(CurJD);
511 if (RISItr != RegisteredInitSymbols.end()) {
512 NewInitSymbols[CurJD] = std::move(RISItr->second);
513 RegisteredInitSymbols.erase(RISItr);
514 }
515
516 for (auto *DepJD : JDDepMap[CurJD])
517 if (Visited.insert(DepJD).second)
518 Worklist.push_back(DepJD);
519 }
520 });
521
522 // If there are no further init symbols to look up then send the link order
523 // (as a list of header addresses) to the caller.
524 if (NewInitSymbols.empty()) {
525 // Build the dep info map to return.
526 COFFJITDylibDepInfoMap DIM;
527 DIM.reserve(JDDepMap.size());
528 for (auto &KV : JDDepMap) {
529 std::lock_guard<std::mutex> Lock(PlatformMutex);
530 COFFJITDylibDepInfo DepInfo;
531 DepInfo.reserve(KV.second.size());
532 for (auto &Dep : KV.second) {
533 DepInfo.push_back(JITDylibToHeaderAddr[Dep]);
534 }
535 auto H = JITDylibToHeaderAddr[KV.first];
536 DIM.push_back(std::make_pair(H, std::move(DepInfo)));
537 }
538 SendResult(DIM);
539 return;
540 }
541
542 // Otherwise issue a lookup and re-run this phase when it completes.
543 lookupInitSymbolsAsync(
544 [this, SendResult = std::move(SendResult), &JD,
545 JDDepMap = std::move(JDDepMap)](Error Err) mutable {
546 if (Err)
547 SendResult(std::move(Err));
548 else
549 pushInitializersLoop(std::move(SendResult), JD, JDDepMap);
550 },
551 ES, std::move(NewInitSymbols));
552 }
553
rt_pushInitializers(PushInitializersSendResultFn SendResult,ExecutorAddr JDHeaderAddr)554 void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
555 ExecutorAddr JDHeaderAddr) {
556 JITDylibSP JD;
557 {
558 std::lock_guard<std::mutex> Lock(PlatformMutex);
559 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
560 if (I != HeaderAddrToJITDylib.end())
561 JD = I->second;
562 }
563
564 LLVM_DEBUG({
565 dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
566 if (JD)
567 dbgs() << "pushing initializers for " << JD->getName() << "\n";
568 else
569 dbgs() << "No JITDylib for header address.\n";
570 });
571
572 if (!JD) {
573 SendResult(make_error<StringError>("No JITDylib with header addr " +
574 formatv("{0:x}", JDHeaderAddr),
575 inconvertibleErrorCode()));
576 return;
577 }
578
579 auto JDDepMap = buildJDDepMap(*JD);
580 if (!JDDepMap) {
581 SendResult(JDDepMap.takeError());
582 return;
583 }
584
585 pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);
586 }
587
rt_lookupSymbol(SendSymbolAddressFn SendResult,ExecutorAddr Handle,StringRef SymbolName)588 void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
589 ExecutorAddr Handle, StringRef SymbolName) {
590 LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
591
592 JITDylib *JD = nullptr;
593
594 {
595 std::lock_guard<std::mutex> Lock(PlatformMutex);
596 auto I = HeaderAddrToJITDylib.find(Handle);
597 if (I != HeaderAddrToJITDylib.end())
598 JD = I->second;
599 }
600
601 if (!JD) {
602 LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n");
603 SendResult(make_error<StringError>("No JITDylib associated with handle " +
604 formatv("{0:x}", Handle),
605 inconvertibleErrorCode()));
606 return;
607 }
608
609 // Use functor class to work around XL build compiler issue on AIX.
610 class RtLookupNotifyComplete {
611 public:
612 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
613 : SendResult(std::move(SendResult)) {}
614 void operator()(Expected<SymbolMap> Result) {
615 if (Result) {
616 assert(Result->size() == 1 && "Unexpected result map count");
617 SendResult(Result->begin()->second.getAddress());
618 } else {
619 SendResult(Result.takeError());
620 }
621 }
622
623 private:
624 SendSymbolAddressFn SendResult;
625 };
626
627 ES.lookup(
628 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
629 SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
630 RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
631 }
632
associateRuntimeSupportFunctions(JITDylib & PlatformJD)633 Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
634 ExecutionSession::JITDispatchHandlerAssociationMap WFs;
635
636 using LookupSymbolSPSSig =
637 SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
638 WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =
639 ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
640 &COFFPlatform::rt_lookupSymbol);
641 using PushInitializersSPSSig =
642 SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
643 WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =
644 ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
645 this, &COFFPlatform::rt_pushInitializers);
646
647 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
648 }
649
runBootstrapInitializers(JDBootstrapState & BState)650 Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
651 llvm::sort(BState.Initializers);
652 if (auto Err =
653 runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))
654 return Err;
655
656 if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))
657 return Err;
658
659 if (auto Err =
660 runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))
661 return Err;
662 return Error::success();
663 }
664
runBootstrapSubsectionInitializers(JDBootstrapState & BState,StringRef Start,StringRef End)665 Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
666 StringRef Start,
667 StringRef End) {
668 for (auto &Initializer : BState.Initializers)
669 if (Initializer.first >= Start && Initializer.first <= End &&
670 Initializer.second) {
671 auto Res =
672 ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);
673 if (!Res)
674 return Res.takeError();
675 }
676 return Error::success();
677 }
678
bootstrapCOFFRuntime(JITDylib & PlatformJD)679 Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
680 // Lookup of runtime symbols causes the collection of initializers if
681 // it's static linking setting.
682 if (auto Err = lookupAndRecordAddrs(
683 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
684 {
685 {ES.intern("__orc_rt_coff_platform_bootstrap"),
686 &orc_rt_coff_platform_bootstrap},
687 {ES.intern("__orc_rt_coff_platform_shutdown"),
688 &orc_rt_coff_platform_shutdown},
689 {ES.intern("__orc_rt_coff_register_jitdylib"),
690 &orc_rt_coff_register_jitdylib},
691 {ES.intern("__orc_rt_coff_deregister_jitdylib"),
692 &orc_rt_coff_deregister_jitdylib},
693 {ES.intern("__orc_rt_coff_register_object_sections"),
694 &orc_rt_coff_register_object_sections},
695 {ES.intern("__orc_rt_coff_deregister_object_sections"),
696 &orc_rt_coff_deregister_object_sections},
697 }))
698 return Err;
699
700 // Call bootstrap functions
701 if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
702 return Err;
703
704 // Do the pending jitdylib registration actions that we couldn't do
705 // because orc runtime was not linked fully.
706 for (auto KV : JDBootstrapStates) {
707 auto &JDBState = KV.second;
708 if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
709 orc_rt_coff_register_jitdylib, JDBState.JDName,
710 JDBState.HeaderAddr))
711 return Err;
712
713 for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
714 if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
715 SPSCOFFObjectSectionsMap, bool)>(
716 orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
717 ObjSectionMap, false))
718 return Err;
719 }
720
721 // Run static initializers collected in bootstrap stage.
722 for (auto KV : JDBootstrapStates) {
723 auto &JDBState = KV.second;
724 if (auto Err = runBootstrapInitializers(JDBState))
725 return Err;
726 }
727
728 return Error::success();
729 }
730
runSymbolIfExists(JITDylib & PlatformJD,StringRef SymbolName)731 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
732 StringRef SymbolName) {
733 ExecutorAddr jit_function;
734 auto AfterCLookupErr = lookupAndRecordAddrs(
735 ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
736 {{ES.intern(SymbolName), &jit_function}});
737 if (!AfterCLookupErr) {
738 auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);
739 if (!Res)
740 return Res.takeError();
741 return Error::success();
742 }
743 if (!AfterCLookupErr.isA<SymbolsNotFound>())
744 return AfterCLookupErr;
745 consumeError(std::move(AfterCLookupErr));
746 return Error::success();
747 }
748
modifyPassConfig(MaterializationResponsibility & MR,jitlink::LinkGraph & LG,jitlink::PassConfiguration & Config)749 void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
750 MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
751 jitlink::PassConfiguration &Config) {
752
753 bool IsBootstrapping = CP.Bootstrapping.load();
754
755 if (auto InitSymbol = MR.getInitializerSymbol()) {
756 if (InitSymbol == CP.COFFHeaderStartSymbol) {
757 Config.PostAllocationPasses.push_back(
758 [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
759 return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);
760 });
761 return;
762 }
763 Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
764 return preserveInitializerSections(G, MR);
765 });
766 }
767
768 if (!IsBootstrapping)
769 Config.PostFixupPasses.push_back(
770 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
771 return registerObjectPlatformSections(G, JD);
772 });
773 else
774 Config.PostFixupPasses.push_back(
775 [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
776 return registerObjectPlatformSectionsInBootstrap(G, JD);
777 });
778 }
779
associateJITDylibHeaderSymbol(jitlink::LinkGraph & G,MaterializationResponsibility & MR,bool IsBootstraping)780 Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
781 jitlink::LinkGraph &G, MaterializationResponsibility &MR,
782 bool IsBootstraping) {
783 auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
784 return *Sym->getName() == *CP.COFFHeaderStartSymbol;
785 });
786 assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
787
788 auto &JD = MR.getTargetJITDylib();
789 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
790 auto HeaderAddr = (*I)->getAddress();
791 CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
792 CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
793 if (!IsBootstraping) {
794 G.allocActions().push_back(
795 {cantFail(WrapperFunctionCall::Create<
796 SPSArgList<SPSString, SPSExecutorAddr>>(
797 CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),
798 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
799 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
800 } else {
801 G.allocActions().push_back(
802 {{},
803 cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
804 CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
805 JDBootstrapState BState;
806 BState.JD = &JD;
807 BState.JDName = JD.getName();
808 BState.HeaderAddr = HeaderAddr;
809 CP.JDBootstrapStates.emplace(&JD, BState);
810 }
811
812 return Error::success();
813 }
814
registerObjectPlatformSections(jitlink::LinkGraph & G,JITDylib & JD)815 Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
816 jitlink::LinkGraph &G, JITDylib &JD) {
817 COFFObjectSectionsMap ObjSecs;
818 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
819 assert(HeaderAddr && "Must be registered jitdylib");
820 for (auto &S : G.sections()) {
821 jitlink::SectionRange Range(S);
822 if (Range.getSize())
823 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
824 }
825
826 G.allocActions().push_back(
827 {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
828 CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),
829 cantFail(
830 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
831 CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
832 ObjSecs))});
833
834 return Error::success();
835 }
836
preserveInitializerSections(jitlink::LinkGraph & G,MaterializationResponsibility & MR)837 Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
838 jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
839
840 if (const auto &InitSymName = MR.getInitializerSymbol()) {
841
842 jitlink::Symbol *InitSym = nullptr;
843
844 for (auto &InitSection : G.sections()) {
845 // Skip non-init sections.
846 if (!isCOFFInitializerSection(InitSection.getName()) ||
847 InitSection.empty())
848 continue;
849
850 // Create the init symbol if it has not been created already and attach it
851 // to the first block.
852 if (!InitSym) {
853 auto &B = **InitSection.blocks().begin();
854 InitSym = &G.addDefinedSymbol(
855 B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
856 jitlink::Scope::SideEffectsOnly, false, true);
857 }
858
859 // Add keep-alive edges to anonymous symbols in all other init blocks.
860 for (auto *B : InitSection.blocks()) {
861 if (B == &InitSym->getBlock())
862 continue;
863
864 auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
865 InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
866 }
867 }
868 }
869
870 return Error::success();
871 }
872
873 Error COFFPlatform::COFFPlatformPlugin::
registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph & G,JITDylib & JD)874 registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
875 JITDylib &JD) {
876 std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
877 auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
878 COFFObjectSectionsMap ObjSecs;
879 for (auto &S : G.sections()) {
880 jitlink::SectionRange Range(S);
881 if (Range.getSize())
882 ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
883 }
884
885 G.allocActions().push_back(
886 {{},
887 cantFail(
888 WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
889 CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
890 ObjSecs))});
891
892 auto &BState = CP.JDBootstrapStates[&JD];
893 BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));
894
895 // Collect static initializers
896 for (auto &S : G.sections())
897 if (isCOFFInitializerSection(S.getName()))
898 for (auto *B : S.blocks()) {
899 if (B->edges_empty())
900 continue;
901 for (auto &E : B->edges())
902 BState.Initializers.push_back(std::make_pair(
903 S.getName().str(), E.getTarget().getAddress() + E.getAddend()));
904 }
905
906 return Error::success();
907 }
908
909 } // End namespace orc.
910 } // End namespace llvm.
911