1 //===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
10
11 #include "llvm/ADT/StringExtras.h"
12 #include "llvm/BinaryFormat/Magic.h"
13 #include "llvm/ExecutionEngine/JITLink/COFF.h"
14 #include "llvm/ExecutionEngine/JITLink/ELF.h"
15 #include "llvm/ExecutionEngine/JITLink/MachO.h"
16 #include "llvm/ExecutionEngine/JITLink/aarch64.h"
17 #include "llvm/ExecutionEngine/JITLink/i386.h"
18 #include "llvm/ExecutionEngine/JITLink/loongarch.h"
19 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
20 #include "llvm/Support/Format.h"
21 #include "llvm/Support/MemoryBuffer.h"
22 #include "llvm/Support/raw_ostream.h"
23
24 using namespace llvm;
25 using namespace llvm::object;
26
27 #define DEBUG_TYPE "jitlink"
28
29 namespace {
30
31 enum JITLinkErrorCode { GenericJITLinkError = 1 };
32
33 // FIXME: This class is only here to support the transition to llvm::Error. It
34 // will be removed once this transition is complete. Clients should prefer to
35 // deal with the Error value directly, rather than converting to error_code.
36 class JITLinkerErrorCategory : public std::error_category {
37 public:
name() const38 const char *name() const noexcept override { return "runtimedyld"; }
39
message(int Condition) const40 std::string message(int Condition) const override {
41 switch (static_cast<JITLinkErrorCode>(Condition)) {
42 case GenericJITLinkError:
43 return "Generic JITLink error";
44 }
45 llvm_unreachable("Unrecognized JITLinkErrorCode");
46 }
47 };
48
49 } // namespace
50
51 namespace llvm {
52 namespace jitlink {
53
54 char JITLinkError::ID = 0;
55
log(raw_ostream & OS) const56 void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; }
57
convertToErrorCode() const58 std::error_code JITLinkError::convertToErrorCode() const {
59 static JITLinkerErrorCategory TheJITLinkerErrorCategory;
60 return std::error_code(GenericJITLinkError, TheJITLinkerErrorCategory);
61 }
62
getGenericEdgeKindName(Edge::Kind K)63 const char *getGenericEdgeKindName(Edge::Kind K) {
64 switch (K) {
65 case Edge::Invalid:
66 return "INVALID RELOCATION";
67 case Edge::KeepAlive:
68 return "Keep-Alive";
69 default:
70 return "<Unrecognized edge kind>";
71 }
72 }
73
getLinkageName(Linkage L)74 const char *getLinkageName(Linkage L) {
75 switch (L) {
76 case Linkage::Strong:
77 return "strong";
78 case Linkage::Weak:
79 return "weak";
80 }
81 llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
82 }
83
getScopeName(Scope S)84 const char *getScopeName(Scope S) {
85 switch (S) {
86 case Scope::Default:
87 return "default";
88 case Scope::Hidden:
89 return "hidden";
90 case Scope::Local:
91 return "local";
92 }
93 llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
94 }
95
isCStringBlock(Block & B)96 bool isCStringBlock(Block &B) {
97 if (B.getSize() == 0) // Empty blocks are not valid C-strings.
98 return false;
99
100 // Zero-fill blocks of size one are valid empty strings.
101 if (B.isZeroFill())
102 return B.getSize() == 1;
103
104 for (size_t I = 0; I != B.getSize() - 1; ++I)
105 if (B.getContent()[I] == '\0')
106 return false;
107
108 return B.getContent()[B.getSize() - 1] == '\0';
109 }
110
operator <<(raw_ostream & OS,const Block & B)111 raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
112 return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize())
113 << ": "
114 << "size = " << formatv("{0:x8}", B.getSize()) << ", "
115 << (B.isZeroFill() ? "zero-fill" : "content")
116 << ", align = " << B.getAlignment()
117 << ", align-ofs = " << B.getAlignmentOffset()
118 << ", section = " << B.getSection().getName();
119 }
120
operator <<(raw_ostream & OS,const Symbol & Sym)121 raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
122 OS << Sym.getAddress() << " (" << (Sym.isDefined() ? "block" : "addressable")
123 << " + " << formatv("{0:x8}", Sym.getOffset())
124 << "): size: " << formatv("{0:x8}", Sym.getSize())
125 << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage()))
126 << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", "
127 << (Sym.isLive() ? "live" : "dead") << " - "
128 << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>");
129 return OS;
130 }
131
printEdge(raw_ostream & OS,const Block & B,const Edge & E,StringRef EdgeKindName)132 void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
133 StringRef EdgeKindName) {
134 OS << "edge@" << B.getAddress() + E.getOffset() << ": " << B.getAddress()
135 << " + " << formatv("{0:x}", E.getOffset()) << " -- " << EdgeKindName
136 << " -> ";
137
138 auto &TargetSym = E.getTarget();
139 if (TargetSym.hasName())
140 OS << TargetSym.getName();
141 else {
142 auto &TargetBlock = TargetSym.getBlock();
143 auto &TargetSec = TargetBlock.getSection();
144 orc::ExecutorAddr SecAddress(~uint64_t(0));
145 for (auto *B : TargetSec.blocks())
146 if (B->getAddress() < SecAddress)
147 SecAddress = B->getAddress();
148
149 orc::ExecutorAddrDiff SecDelta = TargetSym.getAddress() - SecAddress;
150 OS << TargetSym.getAddress() << " (section " << TargetSec.getName();
151 if (SecDelta)
152 OS << " + " << formatv("{0:x}", SecDelta);
153 OS << " / block " << TargetBlock.getAddress();
154 if (TargetSym.getOffset())
155 OS << " + " << formatv("{0:x}", TargetSym.getOffset());
156 OS << ")";
157 }
158
159 if (E.getAddend() != 0)
160 OS << " + " << E.getAddend();
161 }
162
~Section()163 Section::~Section() {
164 for (auto *Sym : Symbols)
165 Sym->~Symbol();
166 for (auto *B : Blocks)
167 B->~Block();
168 }
169
splitBlock(Block & B,size_t SplitIndex,SplitBlockCache * Cache)170 Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex,
171 SplitBlockCache *Cache) {
172
173 assert(SplitIndex > 0 && "splitBlock can not be called with SplitIndex == 0");
174
175 // If the split point covers all of B then just return B.
176 if (SplitIndex == B.getSize())
177 return B;
178
179 assert(SplitIndex < B.getSize() && "SplitIndex out of range");
180
181 // Create the new block covering [ 0, SplitIndex ).
182 auto &NewBlock =
183 B.isZeroFill()
184 ? createZeroFillBlock(B.getSection(), SplitIndex, B.getAddress(),
185 B.getAlignment(), B.getAlignmentOffset())
186 : createContentBlock(
187 B.getSection(), B.getContent().slice(0, SplitIndex),
188 B.getAddress(), B.getAlignment(), B.getAlignmentOffset());
189
190 // Modify B to cover [ SplitIndex, B.size() ).
191 B.setAddress(B.getAddress() + SplitIndex);
192 B.setContent(B.getContent().slice(SplitIndex));
193 B.setAlignmentOffset((B.getAlignmentOffset() + SplitIndex) %
194 B.getAlignment());
195
196 // Handle edge transfer/update.
197 {
198 // Copy edges to NewBlock (recording their iterators so that we can remove
199 // them from B), and update of Edges remaining on B.
200 std::vector<Block::edge_iterator> EdgesToRemove;
201 for (auto I = B.edges().begin(); I != B.edges().end();) {
202 if (I->getOffset() < SplitIndex) {
203 NewBlock.addEdge(*I);
204 I = B.removeEdge(I);
205 } else {
206 I->setOffset(I->getOffset() - SplitIndex);
207 ++I;
208 }
209 }
210 }
211
212 // Handle symbol transfer/update.
213 {
214 // Initialize the symbols cache if necessary.
215 SplitBlockCache LocalBlockSymbolsCache;
216 if (!Cache)
217 Cache = &LocalBlockSymbolsCache;
218 if (*Cache == std::nullopt) {
219 *Cache = SplitBlockCache::value_type();
220 for (auto *Sym : B.getSection().symbols())
221 if (&Sym->getBlock() == &B)
222 (*Cache)->push_back(Sym);
223
224 llvm::sort(**Cache, [](const Symbol *LHS, const Symbol *RHS) {
225 return LHS->getOffset() > RHS->getOffset();
226 });
227 }
228 auto &BlockSymbols = **Cache;
229
230 // Transfer all symbols with offset less than SplitIndex to NewBlock.
231 while (!BlockSymbols.empty() &&
232 BlockSymbols.back()->getOffset() < SplitIndex) {
233 auto *Sym = BlockSymbols.back();
234 // If the symbol extends beyond the split, update the size to be within
235 // the new block.
236 if (Sym->getOffset() + Sym->getSize() > SplitIndex)
237 Sym->setSize(SplitIndex - Sym->getOffset());
238 Sym->setBlock(NewBlock);
239 BlockSymbols.pop_back();
240 }
241
242 // Update offsets for all remaining symbols in B.
243 for (auto *Sym : BlockSymbols)
244 Sym->setOffset(Sym->getOffset() - SplitIndex);
245 }
246
247 return NewBlock;
248 }
249
dump(raw_ostream & OS)250 void LinkGraph::dump(raw_ostream &OS) {
251 DenseMap<Block *, std::vector<Symbol *>> BlockSymbols;
252
253 // Map from blocks to the symbols pointing at them.
254 for (auto *Sym : defined_symbols())
255 BlockSymbols[&Sym->getBlock()].push_back(Sym);
256
257 // For each block, sort its symbols by something approximating
258 // relevance.
259 for (auto &KV : BlockSymbols)
260 llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) {
261 if (LHS->getOffset() != RHS->getOffset())
262 return LHS->getOffset() < RHS->getOffset();
263 if (LHS->getLinkage() != RHS->getLinkage())
264 return LHS->getLinkage() < RHS->getLinkage();
265 if (LHS->getScope() != RHS->getScope())
266 return LHS->getScope() < RHS->getScope();
267 if (LHS->hasName()) {
268 if (!RHS->hasName())
269 return true;
270 return LHS->getName() < RHS->getName();
271 }
272 return false;
273 });
274
275 for (auto &Sec : sections()) {
276 OS << "section " << Sec.getName() << ":\n\n";
277
278 std::vector<Block *> SortedBlocks;
279 llvm::copy(Sec.blocks(), std::back_inserter(SortedBlocks));
280 llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) {
281 return LHS->getAddress() < RHS->getAddress();
282 });
283
284 for (auto *B : SortedBlocks) {
285 OS << " block " << B->getAddress()
286 << " size = " << formatv("{0:x8}", B->getSize())
287 << ", align = " << B->getAlignment()
288 << ", alignment-offset = " << B->getAlignmentOffset();
289 if (B->isZeroFill())
290 OS << ", zero-fill";
291 OS << "\n";
292
293 auto BlockSymsI = BlockSymbols.find(B);
294 if (BlockSymsI != BlockSymbols.end()) {
295 OS << " symbols:\n";
296 auto &Syms = BlockSymsI->second;
297 for (auto *Sym : Syms)
298 OS << " " << *Sym << "\n";
299 } else
300 OS << " no symbols\n";
301
302 if (!B->edges_empty()) {
303 OS << " edges:\n";
304 std::vector<Edge> SortedEdges;
305 llvm::copy(B->edges(), std::back_inserter(SortedEdges));
306 llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) {
307 return LHS.getOffset() < RHS.getOffset();
308 });
309 for (auto &E : SortedEdges) {
310 OS << " " << B->getFixupAddress(E) << " (block + "
311 << formatv("{0:x8}", E.getOffset()) << "), addend = ";
312 if (E.getAddend() >= 0)
313 OS << formatv("+{0:x8}", E.getAddend());
314 else
315 OS << formatv("-{0:x8}", -E.getAddend());
316 OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = ";
317 if (E.getTarget().hasName())
318 OS << E.getTarget().getName();
319 else
320 OS << "addressable@"
321 << formatv("{0:x16}", E.getTarget().getAddress()) << "+"
322 << formatv("{0:x8}", E.getTarget().getOffset());
323 OS << "\n";
324 }
325 } else
326 OS << " no edges\n";
327 OS << "\n";
328 }
329 }
330
331 OS << "Absolute symbols:\n";
332 if (!absolute_symbols().empty()) {
333 for (auto *Sym : absolute_symbols())
334 OS << " " << Sym->getAddress() << ": " << *Sym << "\n";
335 } else
336 OS << " none\n";
337
338 OS << "\nExternal symbols:\n";
339 if (!external_symbols().empty()) {
340 for (auto *Sym : external_symbols())
341 OS << " " << Sym->getAddress() << ": " << *Sym
342 << (Sym->isWeaklyReferenced() ? " (weakly referenced)" : "") << "\n";
343 } else
344 OS << " none\n";
345 }
346
operator <<(raw_ostream & OS,const SymbolLookupFlags & LF)347 raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) {
348 switch (LF) {
349 case SymbolLookupFlags::RequiredSymbol:
350 return OS << "RequiredSymbol";
351 case SymbolLookupFlags::WeaklyReferencedSymbol:
352 return OS << "WeaklyReferencedSymbol";
353 }
354 llvm_unreachable("Unrecognized lookup flags");
355 }
356
anchor()357 void JITLinkAsyncLookupContinuation::anchor() {}
358
359 JITLinkContext::~JITLinkContext() = default;
360
shouldAddDefaultTargetPasses(const Triple & TT) const361 bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
362 return true;
363 }
364
getMarkLivePass(const Triple & TT) const365 LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
366 return LinkGraphPassFunction();
367 }
368
modifyPassConfig(LinkGraph & G,PassConfiguration & Config)369 Error JITLinkContext::modifyPassConfig(LinkGraph &G,
370 PassConfiguration &Config) {
371 return Error::success();
372 }
373
markAllSymbolsLive(LinkGraph & G)374 Error markAllSymbolsLive(LinkGraph &G) {
375 for (auto *Sym : G.defined_symbols())
376 Sym->setLive(true);
377 return Error::success();
378 }
379
makeTargetOutOfRangeError(const LinkGraph & G,const Block & B,const Edge & E)380 Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B,
381 const Edge &E) {
382 std::string ErrMsg;
383 {
384 raw_string_ostream ErrStream(ErrMsg);
385 Section &Sec = B.getSection();
386 ErrStream << "In graph " << G.getName() << ", section " << Sec.getName()
387 << ": relocation target ";
388 if (E.getTarget().hasName()) {
389 ErrStream << "\"" << E.getTarget().getName() << "\"";
390 } else
391 ErrStream << E.getTarget().getBlock().getSection().getName() << " + "
392 << formatv("{0:x}", E.getOffset());
393 ErrStream << " at address " << formatv("{0:x}", E.getTarget().getAddress())
394 << " is out of range of " << G.getEdgeKindName(E.getKind())
395 << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " (";
396
397 Symbol *BestSymbolForBlock = nullptr;
398 for (auto *Sym : Sec.symbols())
399 if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 &&
400 (!BestSymbolForBlock ||
401 Sym->getScope() < BestSymbolForBlock->getScope() ||
402 Sym->getLinkage() < BestSymbolForBlock->getLinkage()))
403 BestSymbolForBlock = Sym;
404
405 if (BestSymbolForBlock)
406 ErrStream << BestSymbolForBlock->getName() << ", ";
407 else
408 ErrStream << "<anonymous block> @ ";
409
410 ErrStream << formatv("{0:x}", B.getAddress()) << " + "
411 << formatv("{0:x}", E.getOffset()) << ")";
412 }
413 return make_error<JITLinkError>(std::move(ErrMsg));
414 }
415
makeAlignmentError(llvm::orc::ExecutorAddr Loc,uint64_t Value,int N,const Edge & E)416 Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N,
417 const Edge &E) {
418 return make_error<JITLinkError>("0x" + llvm::utohexstr(Loc.getValue()) +
419 " improper alignment for relocation " +
420 formatv("{0:d}", E.getKind()) + ": 0x" +
421 llvm::utohexstr(Value) +
422 " is not aligned to " + Twine(N) + " bytes");
423 }
424
getAnonymousPointerCreator(const Triple & TT)425 AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) {
426 switch (TT.getArch()) {
427 case Triple::aarch64:
428 return aarch64::createAnonymousPointer;
429 case Triple::x86_64:
430 return x86_64::createAnonymousPointer;
431 case Triple::x86:
432 return i386::createAnonymousPointer;
433 case Triple::loongarch32:
434 case Triple::loongarch64:
435 return loongarch::createAnonymousPointer;
436 default:
437 return nullptr;
438 }
439 }
440
getPointerJumpStubCreator(const Triple & TT)441 PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) {
442 switch (TT.getArch()) {
443 case Triple::aarch64:
444 return aarch64::createAnonymousPointerJumpStub;
445 case Triple::x86_64:
446 return x86_64::createAnonymousPointerJumpStub;
447 case Triple::x86:
448 return i386::createAnonymousPointerJumpStub;
449 case Triple::loongarch32:
450 case Triple::loongarch64:
451 return loongarch::createAnonymousPointerJumpStub;
452 default:
453 return nullptr;
454 }
455 }
456
457 Expected<std::unique_ptr<LinkGraph>>
createLinkGraphFromObject(MemoryBufferRef ObjectBuffer)458 createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) {
459 auto Magic = identify_magic(ObjectBuffer.getBuffer());
460 switch (Magic) {
461 case file_magic::macho_object:
462 return createLinkGraphFromMachOObject(ObjectBuffer);
463 case file_magic::elf_relocatable:
464 return createLinkGraphFromELFObject(ObjectBuffer);
465 case file_magic::coff_object:
466 return createLinkGraphFromCOFFObject(ObjectBuffer);
467 default:
468 return make_error<JITLinkError>("Unsupported file format");
469 };
470 }
471
absoluteSymbolsLinkGraph(const Triple & TT,orc::SymbolMap Symbols)472 std::unique_ptr<LinkGraph> absoluteSymbolsLinkGraph(const Triple &TT,
473 orc::SymbolMap Symbols) {
474 unsigned PointerSize;
475 endianness Endianness =
476 TT.isLittleEndian() ? endianness::little : endianness::big;
477 switch (TT.getArch()) {
478 case Triple::aarch64:
479 case llvm::Triple::riscv64:
480 case Triple::x86_64:
481 PointerSize = 8;
482 break;
483 case llvm::Triple::arm:
484 case llvm::Triple::riscv32:
485 case llvm::Triple::x86:
486 PointerSize = 4;
487 break;
488 default:
489 llvm::report_fatal_error("unhandled target architecture");
490 }
491
492 static std::atomic<uint64_t> Counter = {0};
493 auto Index = Counter.fetch_add(1, std::memory_order_relaxed);
494 auto G = std::make_unique<LinkGraph>(
495 "<Absolute Symbols " + std::to_string(Index) + ">", TT, PointerSize,
496 Endianness, /*GetEdgeKindName=*/nullptr);
497 for (auto &[Name, Def] : Symbols) {
498 auto &Sym =
499 G->addAbsoluteSymbol(*Name, Def.getAddress(), /*Size=*/0,
500 Linkage::Strong, Scope::Default, /*IsLive=*/true);
501 Sym.setCallable(Def.getFlags().isCallable());
502 }
503
504 return G;
505 }
506
link(std::unique_ptr<LinkGraph> G,std::unique_ptr<JITLinkContext> Ctx)507 void link(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) {
508 switch (G->getTargetTriple().getObjectFormat()) {
509 case Triple::MachO:
510 return link_MachO(std::move(G), std::move(Ctx));
511 case Triple::ELF:
512 return link_ELF(std::move(G), std::move(Ctx));
513 case Triple::COFF:
514 return link_COFF(std::move(G), std::move(Ctx));
515 default:
516 Ctx->notifyFailed(make_error<JITLinkError>("Unsupported object format"));
517 };
518 }
519
520 } // end namespace jitlink
521 } // end namespace llvm
522