xref: /freebsd/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp (revision 95b4436e989df29f6368f13832cb13d7cbc52eac) 
1 //===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
10 
11 #include "llvm/ADT/StringExtras.h"
12 #include "llvm/BinaryFormat/Magic.h"
13 #include "llvm/ExecutionEngine/JITLink/COFF.h"
14 #include "llvm/ExecutionEngine/JITLink/ELF.h"
15 #include "llvm/ExecutionEngine/JITLink/MachO.h"
16 #include "llvm/ExecutionEngine/JITLink/XCOFF.h"
17 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
18 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
19 #include "llvm/ExecutionEngine/JITLink/x86.h"
20 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
21 #include "llvm/Support/raw_ostream.h"
22 
23 using namespace llvm;
24 using namespace llvm::object;
25 
26 #define DEBUG_TYPE "jitlink"
27 
28 namespace {
29 
30 enum JITLinkErrorCode { GenericJITLinkError = 1 };
31 
32 // FIXME: This class is only here to support the transition to llvm::Error. It
33 // will be removed once this transition is complete. Clients should prefer to
34 // deal with the Error value directly, rather than converting to error_code.
35 class JITLinkerErrorCategory : public std::error_category {
36 public:
37   const char *name() const noexcept override { return "runtimedyld"; }
38 
39   std::string message(int Condition) const override {
40     switch (static_cast<JITLinkErrorCode>(Condition)) {
41     case GenericJITLinkError:
42       return "Generic JITLink error";
43     }
44     llvm_unreachable("Unrecognized JITLinkErrorCode");
45   }
46 };
47 
48 } // namespace
49 
50 namespace llvm {
51 namespace jitlink {
52 
53 char JITLinkError::ID = 0;
54 
55 void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
56 
57 std::error_code JITLinkError::convertToErrorCode() const {
58   static JITLinkerErrorCategory TheJITLinkerErrorCategory;
59   return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
60 }
61 
62 const char *getGenericEdgeKindName(Edge::Kind K) {
63   switch (K) {
64   case Edge::Invalid:
65     return "INVALID RELOCATION";
66   case Edge::KeepAlive:
67     return "Keep-Alive";
68   default:
69     return "<Unrecognized edge kind>";
70   }
71 }
72 
73 const char *getLinkageName(Linkage L) {
74   switch (L) {
75   case Linkage::Strong:
76     return "strong";
77   case Linkage::Weak:
78     return "weak";
79   }
80   llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
81 }
82 
83 const char *getScopeName(Scope S) {
84   switch (S) {
85   case Scope::Default:
86     return "default";
87   case Scope::Hidden:
88     return "hidden";
89   case Scope::SideEffectsOnly:
90     return "side-effects-only";
91   case Scope::Local:
92     return "local";
93   }
94   llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
95 }
96 
97 bool isCStringBlock(Block &B) {
98   if (B.getSize() == 0) // Empty blocks are not valid C-strings.
99     return false;
100 
101   // Zero-fill blocks of size one are valid empty strings.
102   if (B.isZeroFill())
103     return B.getSize() == 1;
104 
105   for (size_t I = 0; I != B.getSize() - 1; ++I)
106     if (B.getContent()[I] == '\0')
107       return false;
108 
109   return B.getContent()[B.getSize() - 1] == '\0';
110 }
111 
112 raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
113   return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
114             << ": "
115             << "size = " << formatv("{0:x8}", B.getSize()) << ", "
116             << (B.isZeroFill() ? "zero-fill" : "content")
117             << ", align = " << B.getAlignment()
118             << ", align-ofs = " << B.getAlignmentOffset()
119             << ", section = " << B.getSection().getName();
120 }
121 
122 raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
123   OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
124      << " + " << formatv("{0:x8}", Sym.getOffset())
125      << "): size: " << formatv("{0:x8}", Sym.getSize())
126      << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
127      << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
128      << (Sym.isLive() ? "live" : "dead") << "  -   "
129      << (Sym.hasName() ? *Sym.getName() : "<anonymous symbol>");
130   return OS;
131 }
132 
133 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
134                StringRef EdgeKindName) {
135   OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
136      << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName
137      << " -> ";
138 
139   auto &TargetSym = E.getTarget();
140   if (TargetSym.hasName())
141     OS << TargetSym.getName();
142   else {
143     auto &TargetBlock = TargetSym.getBlock();
144     auto &TargetSec = TargetBlock.getSection();
145     orc::ExecutorAddr SecAddress(~uint64_t(0));
146     for (auto *B : TargetSec.blocks())
147       if (B->getAddress() < SecAddress)
148         SecAddress = B->getAddress();
149 
150     orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
151     OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
152     if (SecDelta)
153       OS << " + " << formatv("{0:x}", SecDelta);
154     OS << " / block " << TargetBlock.getAddress();
155     if (TargetSym.getOffset())
156       OS << " + " << formatv("{0:x}", TargetSym.getOffset());
157     OS << ")";
158   }
159 
160   if (E.getAddend() != 0)
161     OS << " + " << E.getAddend();
162 }
163 
164 Section::~Section() {
165   for (auto *Sym : Symbols)
166     Sym->~Symbol();
167   for (auto *B : Blocks)
168     B->~Block();
169 }
170 
171 LinkGraph::~LinkGraph() {
172   for (auto *Sym : AbsoluteSymbols) {
173     Sym->~Symbol();
174   }
175   for (auto *Sym : external_symbols()) {
176     Sym->~Symbol();
177   }
178   ExternalSymbols.clear();
179 }
180 
181 std::vector<Block *> LinkGraph::splitBlockImpl(std::vector<Block *> Blocks,
182                                                SplitBlockCache *Cache) {
183   assert(!Blocks.empty() && "Blocks must at least contain the original block");
184 
185   // Fix up content of all blocks.
186   ArrayRef<char> Content = Blocks.front()->getContent();
187   for (size_t I = 0; I != Blocks.size() - 1; ++I) {
188     Blocks[I]->setContent(
189         Content.slice(Blocks[I]->getAddress() - Blocks[0]->getAddress(),
190                       Blocks[I + 1]->getAddress() - Blocks[I]->getAddress()));
191   }
192   Blocks.back()->setContent(
193       Content.slice(Blocks.back()->getAddress() - Blocks[0]->getAddress()));
194   bool IsMutable = Blocks[0]->ContentMutable;
195   for (auto *B : Blocks)
196     B->ContentMutable = IsMutable;
197 
198   // Transfer symbols.
199   {
200     SplitBlockCache LocalBlockSymbolsCache;
201     if (!Cache)
202       Cache = &LocalBlockSymbolsCache;
203 
204     // Build cache if required.
205     if (*Cache == std::nullopt) {
206       *Cache = SplitBlockCache::value_type();
207 
208       for (auto *Sym : Blocks[0]->getSection().symbols())
209         if (&Sym->getBlock() == Blocks[0])
210           (*Cache)->push_back(Sym);
211       llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {
212         return LHS->getAddress() > RHS->getAddress();
213       });
214     }
215 
216     auto TransferSymbol = [](Symbol &Sym, Block &B) {
217       Sym.setOffset(Sym.getAddress() - B.getAddress());
218       Sym.setBlock(B);
219       if (Sym.getSize() > B.getSize())
220         Sym.setSize(B.getSize() - Sym.getOffset());
221     };
222 
223     // Transfer symbols to all blocks except the last one.
224     for (size_t I = 0; I != Blocks.size() - 1; ++I) {
225       if ((*Cache)->empty())
226         break;
227       while (!(*Cache)->empty() &&
228              (*Cache)->back()->getAddress() < Blocks[I + 1]->getAddress()) {
229         TransferSymbol(*(*Cache)->back(), *Blocks[I]);
230         (*Cache)->pop_back();
231       }
232     }
233     // Transfer symbols to the last block, checking that all are in-range.
234     while (!(*Cache)->empty()) {
235       auto &Sym = *(*Cache)->back();
236       (*Cache)->pop_back();
237       assert(Sym.getAddress() >= Blocks.back()->getAddress() &&
238              "Symbol address preceeds block");
239       assert(Sym.getAddress() <= Blocks.back()->getRange().End &&
240              "Symbol address starts past end of block");
241       TransferSymbol(Sym, *Blocks.back());
242     }
243   }
244 
245   // Transfer edges.
246   auto &Edges = Blocks[0]->Edges;
247   llvm::sort(Edges, [](const Edge &LHS, const Edge &RHS) {
248     return LHS.getOffset() < RHS.getOffset();
249   });
250 
251   for (size_t I = Blocks.size() - 1; I != 0; --I) {
252 
253     // If all edges have been transferred then bail out.
254     if (Edges.empty())
255       break;
256 
257     Edge::OffsetT Delta = Blocks[I]->getAddress() - Blocks[0]->getAddress();
258 
259     // If no edges to move for this block then move to the next one.
260     if (Edges.back().getOffset() < Delta)
261       continue;
262 
263     size_t EI = Edges.size() - 1;
264     while (EI != 0 && Edges[EI - 1].getOffset() >= Delta)
265       --EI;
266 
267     for (size_t J = EI; J != Edges.size(); ++J) {
268       Blocks[I]->Edges.push_back(std::move(Edges[J]));
269       Blocks[I]->Edges.back().setOffset(Blocks[I]->Edges.back().getOffset() -
270                                         Delta);
271     }
272 
273     while (Edges.size() > EI)
274       Edges.pop_back();
275   }
276 
277   return Blocks;
278 }
279 
280 void LinkGraph::dump(raw_ostream &OS) {
281   DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
282 
283   // Map from blocks to the symbols pointing at them.
284   for (auto *Sym : defined_symbols())
285     BlockSymbols[&Sym->getBlock()].push_back(Sym);
286 
287   // For each block, sort its symbols by something approximating
288   // relevance.
289   for (auto &KV : BlockSymbols)
290     llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {
291       if (LHS->getOffset() != RHS->getOffset())
292         return LHS->getOffset() < RHS->getOffset();
293       if (LHS->getLinkage() != RHS->getLinkage())
294         return LHS->getLinkage() < RHS->getLinkage();
295       if (LHS->getScope() != RHS->getScope())
296         return LHS->getScope() < RHS->getScope();
297       if (LHS->hasName()) {
298         if (!RHS->hasName())
299           return true;
300         return LHS->getName() < RHS->getName();
301       }
302       return false;
303     });
304 
305   std::vector<Section *> SortedSections;
306   for (auto &Sec : sections())
307     SortedSections.push_back(&Sec);
308   llvm::sort(SortedSections, [](const Section *LHS, const Section *RHS) {
309     return LHS->getName() < RHS->getName();
310   });
311 
312   for (auto *Sec : SortedSections) {
313     OS << "section " << Sec->getName() << ":\n\n";
314 
315     std::vector<Block *> SortedBlocks;
316     llvm::append_range(SortedBlocks, Sec->blocks());
317     llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
318       return LHS->getAddress() < RHS->getAddress();
319     });
320 
321     for (auto *B : SortedBlocks) {
322       OS << "  block " << B->getAddress()
323          << " size = " << formatv("{0:x8}", B->getSize())
324          << ", align = " << B->getAlignment()
325          << ", alignment-offset = " << B->getAlignmentOffset();
326       if (B->isZeroFill())
327         OS << ", zero-fill";
328       OS << "\n";
329 
330       auto BlockSymsI = BlockSymbols.find(B);
331       if (BlockSymsI != BlockSymbols.end()) {
332         OS << "    symbols:\n";
333         auto &Syms = BlockSymsI->second;
334         for (auto *Sym : Syms)
335           OS << "      " << *Sym << "\n";
336       } else
337         OS << "    no symbols\n";
338 
339       if (!B->edges_empty()) {
340         OS << "    edges:\n";
341         std::vector<Edge> SortedEdges;
342         llvm::append_range(SortedEdges, B->edges());
343         llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
344           return LHS.getOffset() < RHS.getOffset();
345         });
346         for (auto &E : SortedEdges) {
347           OS << "      " << B->getFixupAddress(E) << " (block + "
348              << formatv("{0:x8}", E.getOffset()) << "), addend = ";
349           if (E.getAddend() >= 0)
350             OS << formatv("+{0:x8}", E.getAddend());
351           else
352             OS << formatv("-{0:x8}", -E.getAddend());
353           OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";
354           if (E.getTarget().hasName())
355             OS << E.getTarget().getName();
356           else
357             OS << "addressable@"
358                << formatv("{0:x16}", E.getTarget().getAddress()) << "+"
359                << formatv("{0:x8}", E.getTarget().getOffset());
360           OS << "\n";
361         }
362       } else
363         OS << "    no edges\n";
364       OS << "\n";
365     }
366   }
367 
368   OS << "Absolute symbols:\n";
369   if (!absolute_symbols().empty()) {
370     for (auto *Sym : absolute_symbols())
371       OS << "  " << Sym->getAddress() << ": " << *Sym << "\n";
372   } else
373     OS << "  none\n";
374 
375   OS << "\nExternal symbols:\n";
376   if (!external_symbols().empty()) {
377     for (auto *Sym : external_symbols())
378       OS << "  " << Sym->getAddress() << ": " << *Sym
379          << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
380   } else
381     OS << "  none\n";
382 }
383 
384 raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
385   switch (LF) {
386   case SymbolLookupFlags::RequiredSymbol:
387     return OS << "RequiredSymbol";
388   case SymbolLookupFlags::WeaklyReferencedSymbol:
389     return OS << "WeaklyReferencedSymbol";
390   }
391   llvm_unreachable("Unrecognized lookup flags");
392 }
393 
394 void JITLinkAsyncLookupContinuation::anchor() {}
395 
396 JITLinkContext::~JITLinkContext() = default;
397 
398 bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
399   return true;
400 }
401 
402 LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
403   return LinkGraphPassFunction();
404 }
405 
406 Error JITLinkContext::modifyPassConfig(LinkGraph &G,
407                                        PassConfiguration &Config) {
408   return Error::success();
409 }
410 
411 Error markAllSymbolsLive(LinkGraph &G) {
412   for (auto *Sym : G.defined_symbols())
413     Sym->setLive(true);
414   return Error::success();
415 }
416 
417 Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
418                                 const Edge &E) {
419   std::string ErrMsg;
420   {
421     raw_string_ostream ErrStream(ErrMsg);
422     Section &Sec = B.getSection();
423     ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
424               << ": relocation target "
425               << formatv("{0:x}", E.getTarget().getAddress() + E.getAddend())
426               << " (";
427     if (E.getTarget().hasName())
428       ErrStream << E.getTarget().getName();
429     else
430       ErrStream << "<anonymous symbol>";
431     if (E.getAddend()) {
432       // Target address includes non-zero added, so break down the arithmetic.
433       ErrStream << formatv(":{0:x}", E.getTarget().getAddress()) << " + "
434                 << formatv("{0:x}", E.getAddend());
435     }
436     ErrStream << ") is out of range of " << G.getEdgeKindName(E.getKind())
437               << " fixup at address "
438               << formatv("{0:x}", E.getTarget().getAddress()) << " (";
439 
440     Symbol *BestSymbolForBlock = nullptr;
441     for (auto *Sym : Sec.symbols())
442       if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
443           (!BestSymbolForBlock ||
444            Sym->getScope() < BestSymbolForBlock->getScope() ||
445            Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
446         BestSymbolForBlock = Sym;
447 
448     if (BestSymbolForBlock)
449       ErrStream << BestSymbolForBlock->getName() << ", ";
450     else
451       ErrStream << "<anonymous block> @ ";
452 
453     ErrStream << formatv("{0:x}", B.getAddress()) << " + "
454               << formatv("{0:x}", E.getOffset()) << ")";
455   }
456   return make_error<JITLinkError>(std::move(ErrMsg));
457 }
458 
459 Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
460                          const Edge &E) {
461   return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +
462                                   " improper alignment for relocation " +
463                                   formatv("{0:d}", E.getKind()) + ": 0x" +
464                                   llvm::utohexstr(Value) +
465                                   " is not aligned to " + Twine(N) + " bytes");
466 }
467 
468 AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
469   switch (TT.getArch()) {
470   case Triple::aarch64:
471     return aarch64::createAnonymousPointer;
472   case Triple::x86_64:
473     return x86_64::createAnonymousPointer;
474   case Triple::x86:
475     return x86::createAnonymousPointer;
476   case Triple::loongarch32:
477   case Triple::loongarch64:
478     return loongarch::createAnonymousPointer;
479   default:
480     return nullptr;
481   }
482 }
483 
484 PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
485   switch (TT.getArch()) {
486   case Triple::aarch64:
487     return aarch64::createAnonymousPointerJumpStub;
488   case Triple::x86_64:
489     return x86_64::createAnonymousPointerJumpStub;
490   case Triple::x86:
491     return x86::createAnonymousPointerJumpStub;
492   case Triple::loongarch32:
493   case Triple::loongarch64:
494     return loongarch::createAnonymousPointerJumpStub;
495   default:
496     return nullptr;
497   }
498 }
499 
500 Expected<std::unique_ptr<LinkGraph>>
501 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer,
502                           std::shared_ptr<orc::SymbolStringPool> SSP) {
503   auto Magic = identify_magic(ObjectBuffer.getBuffer());
504   switch (Magic) {
505   case file_magic::macho_object:
506     return createLinkGraphFromMachOObject(ObjectBuffer, std::move(SSP));
507   case file_magic::elf_relocatable:
508     return createLinkGraphFromELFObject(ObjectBuffer, std::move(SSP));
509   case file_magic::coff_object:
510     return createLinkGraphFromCOFFObject(ObjectBuffer, std::move(SSP));
511   case file_magic::xcoff_object_64:
512     return createLinkGraphFromXCOFFObject(ObjectBuffer, std::move(SSP));
513   default:
514     return make_error<JITLinkError>("Unsupported file format");
515   };
516 }
517 
518 std::unique_ptr<LinkGraph>
519 absoluteSymbolsLinkGraph(Triple TT, std::shared_ptr<orc::SymbolStringPool> SSP,
520                          orc::SymbolMap Symbols) {
521   static std::atomic<uint64_t> Counter = {0};
522   auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
523   auto G = std::make_unique<LinkGraph>(
524       "<Absolute Symbols " + std::to_string(Index) + ">", std::move(SSP),
525       std::move(TT), SubtargetFeatures(), getGenericEdgeKindName);
526   for (auto &[Name, Def] : Symbols) {
527     auto &Sym =
528         G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
529                              Linkage::Strong, Scope::Default, /*IsLive=*/true);
530     Sym.setCallable(Def.getFlags().isCallable());
531   }
532 
533   return G;
534 }
535 
536 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
537   switch (G->getTargetTriple().getObjectFormat()) {
538   case Triple::MachO:
539     return link_MachO(std::move(G), std::move(Ctx));
540   case Triple::ELF:
541     return link_ELF(std::move(G), std::move(Ctx));
542   case Triple::COFF:
543     return link_COFF(std::move(G), std::move(Ctx));
544   case Triple::XCOFF:
545     return link_XCOFF(std::move(G), std::move(Ctx));
546   default:
547     Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));
548   };
549 }
550 
551 } // end namespace jitlink
552 } // end namespace llvm
553