xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp (revision e3f4a63af63bea70bc86b6c790b14aa5ee99fcd0) 
1 //===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/Orc/COFFPlatform.h"
10 
11 #include "llvm/ExecutionEngine/Orc/AbsoluteSymbols.h"
12 #include "llvm/ExecutionEngine/Orc/COFF.h"
13 #include "llvm/ExecutionEngine/Orc/DebugUtils.h"
14 #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h"
15 #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h"
16 #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h"
17 
18 #include "llvm/Object/COFF.h"
19 
20 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
21 
22 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
23 
24 #define DEBUG_TYPE "orc"
25 
26 using namespace llvm;
27 using namespace llvm::orc;
28 using namespace llvm::orc::shared;
29 
30 namespace llvm {
31 namespace orc {
32 namespace shared {
33 
34 using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>;
35 using SPSCOFFJITDylibDepInfoMap =
36     SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>;
37 using SPSCOFFObjectSectionsMap =
38     SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>;
39 using SPSCOFFRegisterObjectSectionsArgs =
40     SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>;
41 using SPSCOFFDeregisterObjectSectionsArgs =
42     SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>;
43 
44 } // namespace shared
45 } // namespace orc
46 } // namespace llvm
47 namespace {
48 
49 class COFFHeaderMaterializationUnit : public MaterializationUnit {
50 public:
51   COFFHeaderMaterializationUnit(COFFPlatform &CP,
52                                 const SymbolStringPtr &HeaderStartSymbol)
53       : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)),
54         CP(CP) {}
55 
56   StringRef getName() const override { return "COFFHeaderMU"; }
57 
58   void materialize(std::unique_ptr<MaterializationResponsibility> R) override {
59     auto G = std::make_unique<jitlink::LinkGraph>(
60         "<COFFHeaderMU>", CP.getExecutionSession().getSymbolStringPool(),
61         CP.getExecutionSession().getTargetTriple(), SubtargetFeatures(),
62         jitlink::getGenericEdgeKindName);
63     auto &HeaderSection = G->createSection("__header", MemProt::Read);
64     auto &HeaderBlock = createHeaderBlock(*G, HeaderSection);
65 
66     // Init symbol is __ImageBase symbol.
67     auto &ImageBaseSymbol = G->addDefinedSymbol(
68         HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(),
69         jitlink::Linkage::Strong, jitlink::Scope::Default, false, true);
70 
71     addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol);
72 
73     CP.getObjectLinkingLayer().emit(std::move(R), std::move(G));
74   }
75 
76   void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {}
77 
78 private:
79   struct HeaderSymbol {
80     const char *Name;
81     uint64_t Offset;
82   };
83 
84   struct NTHeader {
85     support::ulittle32_t PEMagic;
86     object::coff_file_header FileHeader;
87     struct PEHeader {
88       object::pe32plus_header Header;
89       object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1];
90     } OptionalHeader;
91   };
92 
93   struct HeaderBlockContent {
94     object::dos_header DOSHeader;
95     COFFHeaderMaterializationUnit::NTHeader NTHeader;
96   };
97 
98   static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G,
99                                            jitlink::Section &HeaderSection) {
100     HeaderBlockContent Hdr = {};
101 
102     // Set up magic
103     Hdr.DOSHeader.Magic[0] = 'M';
104     Hdr.DOSHeader.Magic[1] = 'Z';
105     Hdr.DOSHeader.AddressOfNewExeHeader =
106         offsetof(HeaderBlockContent, NTHeader);
107     uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic);
108     Hdr.NTHeader.PEMagic = PEMagic;
109     Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS;
110 
111     switch (G.getTargetTriple().getArch()) {
112     case Triple::x86_64:
113       Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64;
114       break;
115     default:
116       llvm_unreachable("Unrecognized architecture");
117     }
118 
119     auto HeaderContent = G.allocateContent(
120         ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr)));
121 
122     return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8,
123                                 0);
124   }
125 
126   static void addImageBaseRelocationEdge(jitlink::Block &B,
127                                          jitlink::Symbol &ImageBase) {
128     auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) +
129                            offsetof(NTHeader, OptionalHeader) +
130                            offsetof(object::pe32plus_header, ImageBase);
131     B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0);
132   }
133 
134   static MaterializationUnit::Interface
135   createHeaderInterface(COFFPlatform &MOP,
136                         const SymbolStringPtr &HeaderStartSymbol) {
137     SymbolFlagsMap HeaderSymbolFlags;
138 
139     HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported;
140 
141     return MaterializationUnit::Interface(std::move(HeaderSymbolFlags),
142                                           HeaderStartSymbol);
143   }
144 
145   COFFPlatform &CP;
146 };
147 
148 } // end anonymous namespace
149 
150 namespace llvm {
151 namespace orc {
152 
153 Expected<std::unique_ptr<COFFPlatform>>
154 COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
155                      std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
156                      LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
157                      const char *VCRuntimePath,
158                      std::optional<SymbolAliasMap> RuntimeAliases) {
159 
160   auto &ES = ObjLinkingLayer.getExecutionSession();
161 
162   // If the target is not supported then bail out immediately.
163   if (!supportedTarget(ES.getTargetTriple()))
164     return make_error<StringError>("Unsupported COFFPlatform triple: " +
165                                        ES.getTargetTriple().str(),
166                                    inconvertibleErrorCode());
167 
168   auto &EPC = ES.getExecutorProcessControl();
169 
170   auto GeneratorArchive =
171       object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef());
172   if (!GeneratorArchive)
173     return GeneratorArchive.takeError();
174 
175   std::set<std::string> DylibsToPreload;
176   auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create(
177       ObjLinkingLayer, nullptr, std::move(*GeneratorArchive),
178       COFFImportFileScanner(DylibsToPreload));
179   if (!OrcRuntimeArchiveGenerator)
180     return OrcRuntimeArchiveGenerator.takeError();
181 
182   // We need a second instance of the archive (for now) for the Platform. We
183   // can `cantFail` this call, since if it were going to fail it would have
184   // failed above.
185   auto RuntimeArchive = cantFail(
186       object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()));
187 
188   // Create default aliases if the caller didn't supply any.
189   if (!RuntimeAliases)
190     RuntimeAliases = standardPlatformAliases(ES);
191 
192   // Define the aliases.
193   if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases))))
194     return std::move(Err);
195 
196   auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>");
197 
198   // Add JIT-dispatch function support symbols.
199   if (auto Err = HostFuncJD.define(
200           absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"),
201                             {EPC.getJITDispatchInfo().JITDispatchFunction,
202                              JITSymbolFlags::Exported}},
203                            {ES.intern("__orc_rt_jit_dispatch_ctx"),
204                             {EPC.getJITDispatchInfo().JITDispatchContext,
205                              JITSymbolFlags::Exported}}})))
206     return std::move(Err);
207 
208   PlatformJD.addToLinkOrder(HostFuncJD);
209 
210   // Create the instance.
211   Error Err = Error::success();
212   auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform(
213       ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator),
214       std::move(DylibsToPreload), std::move(OrcRuntimeArchiveBuffer),
215       std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime,
216       VCRuntimePath, Err));
217   if (Err)
218     return std::move(Err);
219   return std::move(P);
220 }
221 
222 Expected<std::unique_ptr<COFFPlatform>>
223 COFFPlatform::Create(ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
224                      const char *OrcRuntimePath,
225                      LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
226                      const char *VCRuntimePath,
227                      std::optional<SymbolAliasMap> RuntimeAliases) {
228 
229   auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath);
230   if (!ArchiveBuffer)
231     return createFileError(OrcRuntimePath, ArchiveBuffer.getError());
232 
233   return Create(ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer),
234                 std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath,
235                 std::move(RuntimeAliases));
236 }
237 
238 Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() {
239   auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker");
240   if (!PerJDObj)
241     return PerJDObj.takeError();
242 
243   if (!*PerJDObj)
244     return make_error<StringError>("Could not find per jd object file",
245                                    inconvertibleErrorCode());
246 
247   auto Buffer = (*PerJDObj)->getAsBinary();
248   if (!Buffer)
249     return Buffer.takeError();
250 
251   return (*Buffer)->getMemoryBufferRef();
252 }
253 
254 static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases,
255                        ArrayRef<std::pair<const char *, const char *>> AL) {
256   for (auto &KV : AL) {
257     auto AliasName = ES.intern(KV.first);
258     assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map");
259     Aliases[std::move(AliasName)] = {ES.intern(KV.second),
260                                      JITSymbolFlags::Exported};
261   }
262 }
263 
264 Error COFFPlatform::setupJITDylib(JITDylib &JD) {
265   if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>(
266           *this, COFFHeaderStartSymbol)))
267     return Err;
268 
269   if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError())
270     return Err;
271 
272   // Define the CXX aliases.
273   SymbolAliasMap CXXAliases;
274   addAliases(ES, CXXAliases, requiredCXXAliases());
275   if (auto Err = JD.define(symbolAliases(std::move(CXXAliases))))
276     return Err;
277 
278   auto PerJDObj = getPerJDObjectFile();
279   if (!PerJDObj)
280     return PerJDObj.takeError();
281 
282   auto I = getObjectFileInterface(ES, *PerJDObj);
283   if (!I)
284     return I.takeError();
285 
286   if (auto Err = ObjLinkingLayer.add(
287           JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I)))
288     return Err;
289 
290   if (!Bootstrapping) {
291     auto ImportedLibs = StaticVCRuntime
292                             ? VCRuntimeBootstrap->loadStaticVCRuntime(JD)
293                             : VCRuntimeBootstrap->loadDynamicVCRuntime(JD);
294     if (!ImportedLibs)
295       return ImportedLibs.takeError();
296     for (auto &Lib : *ImportedLibs)
297       if (auto Err = LoadDynLibrary(JD, Lib))
298         return Err;
299     if (StaticVCRuntime)
300       if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD))
301         return Err;
302   }
303 
304   JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer));
305   return Error::success();
306 }
307 
308 Error COFFPlatform::teardownJITDylib(JITDylib &JD) {
309   std::lock_guard<std::mutex> Lock(PlatformMutex);
310   auto I = JITDylibToHeaderAddr.find(&JD);
311   if (I != JITDylibToHeaderAddr.end()) {
312     assert(HeaderAddrToJITDylib.count(I->second) &&
313            "HeaderAddrToJITDylib missing entry");
314     HeaderAddrToJITDylib.erase(I->second);
315     JITDylibToHeaderAddr.erase(I);
316   }
317   return Error::success();
318 }
319 
320 Error COFFPlatform::notifyAdding(ResourceTracker &RT,
321                                  const MaterializationUnit &MU) {
322   auto &JD = RT.getJITDylib();
323   const auto &InitSym = MU.getInitializerSymbol();
324   if (!InitSym)
325     return Error::success();
326 
327   RegisteredInitSymbols[&JD].add(InitSym,
328                                  SymbolLookupFlags::WeaklyReferencedSymbol);
329 
330   LLVM_DEBUG({
331     dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU "
332            << MU.getName() << "\n";
333   });
334   return Error::success();
335 }
336 
337 Error COFFPlatform::notifyRemoving(ResourceTracker &RT) {
338   llvm_unreachable("Not supported yet");
339 }
340 
341 SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) {
342   SymbolAliasMap Aliases;
343   addAliases(ES, Aliases, standardRuntimeUtilityAliases());
344   return Aliases;
345 }
346 
347 ArrayRef<std::pair<const char *, const char *>>
348 COFFPlatform::requiredCXXAliases() {
349   static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
350       {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"},
351       {"_onexit", "__orc_rt_coff_onexit_per_jd"},
352       {"atexit", "__orc_rt_coff_atexit_per_jd"}};
353 
354   return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases);
355 }
356 
357 ArrayRef<std::pair<const char *, const char *>>
358 COFFPlatform::standardRuntimeUtilityAliases() {
359   static const std::pair<const char *, const char *>
360       StandardRuntimeUtilityAliases[] = {
361           {"__orc_rt_run_program", "__orc_rt_coff_run_program"},
362           {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"},
363           {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"},
364           {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"},
365           {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"},
366           {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}};
367 
368   return ArrayRef<std::pair<const char *, const char *>>(
369       StandardRuntimeUtilityAliases);
370 }
371 
372 bool COFFPlatform::supportedTarget(const Triple &TT) {
373   switch (TT.getArch()) {
374   case Triple::x86_64:
375     return true;
376   default:
377     return false;
378   }
379 }
380 
381 COFFPlatform::COFFPlatform(
382     ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD,
383     std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator,
384     std::set<std::string> DylibsToPreload,
385     std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer,
386     std::unique_ptr<object::Archive> OrcRuntimeArchive,
387     LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime,
388     const char *VCRuntimePath, Error &Err)
389     : ES(ObjLinkingLayer.getExecutionSession()),
390       ObjLinkingLayer(ObjLinkingLayer),
391       LoadDynLibrary(std::move(LoadDynLibrary)),
392       OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)),
393       OrcRuntimeArchive(std::move(OrcRuntimeArchive)),
394       StaticVCRuntime(StaticVCRuntime),
395       COFFHeaderStartSymbol(ES.intern("__ImageBase")) {
396   ErrorAsOutParameter _(Err);
397 
398   Bootstrapping.store(true);
399   ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this));
400 
401   // Load vc runtime
402   auto VCRT =
403       COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath);
404   if (!VCRT) {
405     Err = VCRT.takeError();
406     return;
407   }
408   VCRuntimeBootstrap = std::move(*VCRT);
409 
410   auto ImportedLibs =
411       StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD)
412                       : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD);
413   if (!ImportedLibs) {
414     Err = ImportedLibs.takeError();
415     return;
416   }
417 
418   for (auto &Lib : *ImportedLibs)
419     DylibsToPreload.insert(Lib);
420 
421   PlatformJD.addGenerator(std::move(OrcRuntimeGenerator));
422 
423   // PlatformJD hasn't been set up by the platform yet (since we're creating
424   // the platform now), so set it up.
425   if (auto E2 = setupJITDylib(PlatformJD)) {
426     Err = std::move(E2);
427     return;
428   }
429 
430   for (auto& Lib : DylibsToPreload)
431     if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) {
432       Err = std::move(E2);
433       return;
434     }
435 
436   if (StaticVCRuntime)
437       if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) {
438           Err = std::move(E2);
439           return;
440       }
441 
442   // Associate wrapper function tags with JIT-side function implementations.
443   if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
444       Err = std::move(E2);
445       return;
446   }
447 
448   // Lookup addresses of runtime functions callable by the platform,
449   // call the platform bootstrap function to initialize the platform-state
450   // object in the executor.
451   if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) {
452       Err = std::move(E2);
453       return;
454   }
455 
456   Bootstrapping.store(false);
457   JDBootstrapStates.clear();
458 }
459 
460 Expected<COFFPlatform::JITDylibDepMap>
461 COFFPlatform::buildJDDepMap(JITDylib &JD) {
462   return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> {
463     JITDylibDepMap JDDepMap;
464 
465     SmallVector<JITDylib *, 16> Worklist({&JD});
466     while (!Worklist.empty()) {
467       auto CurJD = Worklist.back();
468       Worklist.pop_back();
469 
470       auto &DM = JDDepMap[CurJD];
471       CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
472         DM.reserve(O.size());
473         for (auto &KV : O) {
474           if (KV.first == CurJD)
475             continue;
476           {
477             // Bare jitdylibs not known to the platform
478             std::lock_guard<std::mutex> Lock(PlatformMutex);
479             if (!JITDylibToHeaderAddr.count(KV.first)) {
480               LLVM_DEBUG({
481                 dbgs() << "JITDylib unregistered to COFFPlatform detected in "
482                           "LinkOrder: "
483                        << CurJD->getName() << "\n";
484               });
485               continue;
486             }
487           }
488           DM.push_back(KV.first);
489           // Push unvisited entry.
490           if (JDDepMap.try_emplace(KV.first).second)
491             Worklist.push_back(KV.first);
492         }
493       });
494     }
495     return std::move(JDDepMap);
496   });
497 }
498 
499 void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult,
500                                         JITDylibSP JD,
501                                         JITDylibDepMap &JDDepMap) {
502   SmallVector<JITDylib *, 16> Worklist({JD.get()});
503   DenseSet<JITDylib *> Visited({JD.get()});
504   DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols;
505   ES.runSessionLocked([&]() {
506     while (!Worklist.empty()) {
507       auto CurJD = Worklist.back();
508       Worklist.pop_back();
509 
510       auto RISItr = RegisteredInitSymbols.find(CurJD);
511       if (RISItr != RegisteredInitSymbols.end()) {
512         NewInitSymbols[CurJD] = std::move(RISItr->second);
513         RegisteredInitSymbols.erase(RISItr);
514       }
515 
516       for (auto *DepJD : JDDepMap[CurJD])
517         if (Visited.insert(DepJD).second)
518           Worklist.push_back(DepJD);
519     }
520   });
521 
522   // If there are no further init symbols to look up then send the link order
523   // (as a list of header addresses) to the caller.
524   if (NewInitSymbols.empty()) {
525     // Build the dep info map to return.
526     COFFJITDylibDepInfoMap DIM;
527     DIM.reserve(JDDepMap.size());
528     for (auto &KV : JDDepMap) {
529       std::lock_guard<std::mutex> Lock(PlatformMutex);
530       COFFJITDylibDepInfo DepInfo;
531       DepInfo.reserve(KV.second.size());
532       for (auto &Dep : KV.second) {
533         DepInfo.push_back(JITDylibToHeaderAddr[Dep]);
534       }
535       auto H = JITDylibToHeaderAddr[KV.first];
536       DIM.push_back(std::make_pair(H, std::move(DepInfo)));
537     }
538     SendResult(DIM);
539     return;
540   }
541 
542   // Otherwise issue a lookup and re-run this phase when it completes.
543   lookupInitSymbolsAsync(
544       [this, SendResult = std::move(SendResult), &JD,
545        JDDepMap = std::move(JDDepMap)](Error Err) mutable {
546         if (Err)
547           SendResult(std::move(Err));
548         else
549           pushInitializersLoop(std::move(SendResult), JD, JDDepMap);
550       },
551       ES, std::move(NewInitSymbols));
552 }
553 
554 void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
555                                        ExecutorAddr JDHeaderAddr) {
556   JITDylibSP JD;
557   {
558     std::lock_guard<std::mutex> Lock(PlatformMutex);
559     auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
560     if (I != HeaderAddrToJITDylib.end())
561       JD = I->second;
562   }
563 
564   LLVM_DEBUG({
565     dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") ";
566     if (JD)
567       dbgs() << "pushing initializers for " << JD->getName() << "\n";
568     else
569       dbgs() << "No JITDylib for header address.\n";
570   });
571 
572   if (!JD) {
573     SendResult(make_error<StringError>("No JITDylib with header addr " +
574                                            formatv("{0:x}", JDHeaderAddr),
575                                        inconvertibleErrorCode()));
576     return;
577   }
578 
579   auto JDDepMap = buildJDDepMap(*JD);
580   if (!JDDepMap) {
581     SendResult(JDDepMap.takeError());
582     return;
583   }
584 
585   pushInitializersLoop(std::move(SendResult), JD, *JDDepMap);
586 }
587 
588 void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
589                                    ExecutorAddr Handle, StringRef SymbolName) {
590   LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n");
591 
592   JITDylib *JD = nullptr;
593 
594   {
595     std::lock_guard<std::mutex> Lock(PlatformMutex);
596     auto I = HeaderAddrToJITDylib.find(Handle);
597     if (I != HeaderAddrToJITDylib.end())
598       JD = I->second;
599   }
600 
601   if (!JD) {
602     LLVM_DEBUG(dbgs() << "  No JITDylib for handle " << Handle << "\n");
603     SendResult(make_error<StringError>("No JITDylib associated with handle " +
604                                            formatv("{0:x}", Handle),
605                                        inconvertibleErrorCode()));
606     return;
607   }
608 
609   // Use functor class to work around XL build compiler issue on AIX.
610   class RtLookupNotifyComplete {
611   public:
612     RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
613         : SendResult(std::move(SendResult)) {}
614     void operator()(Expected<SymbolMap> Result) {
615       if (Result) {
616         assert(Result->size() == 1 && "Unexpected result map count");
617         SendResult(Result->begin()->second.getAddress());
618       } else {
619         SendResult(Result.takeError());
620       }
621     }
622 
623   private:
624     SendSymbolAddressFn SendResult;
625   };
626 
627   ES.lookup(
628       LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
629       SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready,
630       RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister);
631 }
632 
633 Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) {
634   ExecutionSession::JITDispatchHandlerAssociationMap WFs;
635 
636   using LookupSymbolSPSSig =
637       SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString);
638   WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] =
639       ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this,
640                                               &COFFPlatform::rt_lookupSymbol);
641   using PushInitializersSPSSig =
642       SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr);
643   WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] =
644       ES.wrapAsyncWithSPS<PushInitializersSPSSig>(
645           this, &COFFPlatform::rt_pushInitializers);
646 
647   return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
648 }
649 
650 Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) {
651   llvm::sort(BState.Initializers);
652   if (auto Err =
653           runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ"))
654     return Err;
655 
656   if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init"))
657     return Err;
658 
659   if (auto Err =
660           runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ"))
661     return Err;
662   return Error::success();
663 }
664 
665 Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState,
666                                                        StringRef Start,
667                                                        StringRef End) {
668   for (auto &Initializer : BState.Initializers)
669     if (Initializer.first >= Start && Initializer.first <= End &&
670         Initializer.second) {
671       auto Res =
672           ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second);
673       if (!Res)
674         return Res.takeError();
675     }
676   return Error::success();
677 }
678 
679 Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) {
680   // Lookup of runtime symbols causes the collection of initializers if
681   // it's static linking setting.
682   if (auto Err = lookupAndRecordAddrs(
683           ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
684           {
685               {ES.intern("__orc_rt_coff_platform_bootstrap"),
686                &orc_rt_coff_platform_bootstrap},
687               {ES.intern("__orc_rt_coff_platform_shutdown"),
688                &orc_rt_coff_platform_shutdown},
689               {ES.intern("__orc_rt_coff_register_jitdylib"),
690                &orc_rt_coff_register_jitdylib},
691               {ES.intern("__orc_rt_coff_deregister_jitdylib"),
692                &orc_rt_coff_deregister_jitdylib},
693               {ES.intern("__orc_rt_coff_register_object_sections"),
694                &orc_rt_coff_register_object_sections},
695               {ES.intern("__orc_rt_coff_deregister_object_sections"),
696                &orc_rt_coff_deregister_object_sections},
697           }))
698     return Err;
699 
700   // Call bootstrap functions
701   if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap))
702     return Err;
703 
704   // Do the pending jitdylib registration actions that we couldn't do
705   // because orc runtime was not linked fully.
706   for (auto KV : JDBootstrapStates) {
707     auto &JDBState = KV.second;
708     if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>(
709             orc_rt_coff_register_jitdylib, JDBState.JDName,
710             JDBState.HeaderAddr))
711       return Err;
712 
713     for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps)
714       if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr,
715                                             SPSCOFFObjectSectionsMap, bool)>(
716               orc_rt_coff_register_object_sections, JDBState.HeaderAddr,
717               ObjSectionMap, false))
718         return Err;
719   }
720 
721   // Run static initializers collected in bootstrap stage.
722   for (auto KV : JDBootstrapStates) {
723     auto &JDBState = KV.second;
724     if (auto Err = runBootstrapInitializers(JDBState))
725       return Err;
726   }
727 
728   return Error::success();
729 }
730 
731 Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD,
732                                       StringRef SymbolName) {
733   ExecutorAddr jit_function;
734   auto AfterCLookupErr = lookupAndRecordAddrs(
735       ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD),
736       {{ES.intern(SymbolName), &jit_function}});
737   if (!AfterCLookupErr) {
738     auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function);
739     if (!Res)
740       return Res.takeError();
741     return Error::success();
742   }
743   if (!AfterCLookupErr.isA<SymbolsNotFound>())
744     return AfterCLookupErr;
745   consumeError(std::move(AfterCLookupErr));
746   return Error::success();
747 }
748 
749 void COFFPlatform::COFFPlatformPlugin::modifyPassConfig(
750     MaterializationResponsibility &MR, jitlink::LinkGraph &LG,
751     jitlink::PassConfiguration &Config) {
752 
753   bool IsBootstrapping = CP.Bootstrapping.load();
754 
755   if (auto InitSymbol = MR.getInitializerSymbol()) {
756     if (InitSymbol == CP.COFFHeaderStartSymbol) {
757       Config.PostAllocationPasses.push_back(
758           [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) {
759             return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping);
760           });
761       return;
762     }
763     Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) {
764       return preserveInitializerSections(G, MR);
765     });
766   }
767 
768   if (!IsBootstrapping)
769     Config.PostFixupPasses.push_back(
770         [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
771           return registerObjectPlatformSections(G, JD);
772         });
773   else
774     Config.PostFixupPasses.push_back(
775         [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) {
776           return registerObjectPlatformSectionsInBootstrap(G, JD);
777         });
778 }
779 
780 Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol(
781     jitlink::LinkGraph &G, MaterializationResponsibility &MR,
782     bool IsBootstraping) {
783   auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) {
784     return *Sym->getName() == *CP.COFFHeaderStartSymbol;
785   });
786   assert(I != G.defined_symbols().end() && "Missing COFF header start symbol");
787 
788   auto &JD = MR.getTargetJITDylib();
789   std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
790   auto HeaderAddr = (*I)->getAddress();
791   CP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
792   CP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
793   if (!IsBootstraping) {
794     G.allocActions().push_back(
795         {cantFail(WrapperFunctionCall::Create<
796                   SPSArgList<SPSString, SPSExecutorAddr>>(
797              CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)),
798          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
799              CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
800   } else {
801     G.allocActions().push_back(
802         {{},
803          cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>(
804              CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))});
805     JDBootstrapState BState;
806     BState.JD = &JD;
807     BState.JDName = JD.getName();
808     BState.HeaderAddr = HeaderAddr;
809     CP.JDBootstrapStates.emplace(&JD, BState);
810   }
811 
812   return Error::success();
813 }
814 
815 Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections(
816     jitlink::LinkGraph &G, JITDylib &JD) {
817   COFFObjectSectionsMap ObjSecs;
818   auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
819   assert(HeaderAddr && "Must be registered jitdylib");
820   for (auto &S : G.sections()) {
821     jitlink::SectionRange Range(S);
822     if (Range.getSize())
823       ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
824   }
825 
826   G.allocActions().push_back(
827       {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>(
828            CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)),
829        cantFail(
830            WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
831                CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
832                ObjSecs))});
833 
834   return Error::success();
835 }
836 
837 Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections(
838     jitlink::LinkGraph &G, MaterializationResponsibility &MR) {
839 
840   if (const auto &InitSymName = MR.getInitializerSymbol()) {
841 
842     jitlink::Symbol *InitSym = nullptr;
843 
844     for (auto &InitSection : G.sections()) {
845       // Skip non-init sections.
846       if (!isCOFFInitializerSection(InitSection.getName()) ||
847           InitSection.empty())
848         continue;
849 
850       // Create the init symbol if it has not been created already and attach it
851       // to the first block.
852       if (!InitSym) {
853         auto &B = **InitSection.blocks().begin();
854         InitSym = &G.addDefinedSymbol(
855             B, 0, *InitSymName, B.getSize(), jitlink::Linkage::Strong,
856             jitlink::Scope::SideEffectsOnly, false, true);
857       }
858 
859       // Add keep-alive edges to anonymous symbols in all other init blocks.
860       for (auto *B : InitSection.blocks()) {
861         if (B == &InitSym->getBlock())
862           continue;
863 
864         auto &S = G.addAnonymousSymbol(*B, 0, B->getSize(), false, true);
865         InitSym->getBlock().addEdge(jitlink::Edge::KeepAlive, 0, S, 0);
866       }
867     }
868   }
869 
870   return Error::success();
871 }
872 
873 Error COFFPlatform::COFFPlatformPlugin::
874     registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G,
875                                               JITDylib &JD) {
876   std::lock_guard<std::mutex> Lock(CP.PlatformMutex);
877   auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD];
878   COFFObjectSectionsMap ObjSecs;
879   for (auto &S : G.sections()) {
880     jitlink::SectionRange Range(S);
881     if (Range.getSize())
882       ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange()));
883   }
884 
885   G.allocActions().push_back(
886       {{},
887        cantFail(
888            WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>(
889                CP.orc_rt_coff_deregister_object_sections, HeaderAddr,
890                ObjSecs))});
891 
892   auto &BState = CP.JDBootstrapStates[&JD];
893   BState.ObjectSectionsMaps.push_back(std::move(ObjSecs));
894 
895   // Collect static initializers
896   for (auto &S : G.sections())
897     if (isCOFFInitializerSection(S.getName()))
898       for (auto *B : S.blocks()) {
899         if (B->edges_empty())
900           continue;
901         for (auto &E : B->edges())
902           BState.Initializers.push_back(std::make_pair(
903               S.getName().str(), E.getTarget().getAddress() + E.getAddend()));
904       }
905 
906   return Error::success();
907 }
908 
909 } // End namespace orc.
910 } // End namespace llvm.
911