1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // ELF/x86-64 jit-link implementation.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h"
15 #include "llvm/ExecutionEngine/JITLink/JITLink.h"
16 #include "llvm/ExecutionEngine/JITLink/TableManager.h"
17 #include "llvm/ExecutionEngine/JITLink/x86_64.h"
18 #include "llvm/Object/ELFObjectFile.h"
19
20 #include "DefineExternalSectionStartAndEndSymbols.h"
21 #include "EHFrameSupportImpl.h"
22 #include "ELFLinkGraphBuilder.h"
23 #include "JITLinkGeneric.h"
24
25 #define DEBUG_TYPE "jitlink"
26
27 using namespace llvm;
28 using namespace llvm::jitlink;
29
30 namespace {
31
32 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_";
33 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO";
34
35 class TLSInfoTableManager_ELF_x86_64
36 : public TableManager<TLSInfoTableManager_ELF_x86_64> {
37 public:
38 static const uint8_t TLSInfoEntryContent[16];
39
getSectionName()40 static StringRef getSectionName() { return ELFTLSInfoSectionName; }
41
visitEdge(LinkGraph & G,Block * B,Edge & E)42 bool visitEdge(LinkGraph &G, Block *B, Edge &E) {
43 if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) {
44 LLVM_DEBUG({
45 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
46 << formatv("{0:x}", B->getFixupAddress(E)) << " ("
47 << formatv("{0:x}", B->getAddress()) << " + "
48 << formatv("{0:x}", E.getOffset()) << ")\n";
49 });
50 E.setKind(x86_64::Delta32);
51 E.setTarget(getEntryForTarget(G, E.getTarget()));
52 return true;
53 }
54 return false;
55 }
56
createEntry(LinkGraph & G,Symbol & Target)57 Symbol &createEntry(LinkGraph &G, Symbol &Target) {
58 // the TLS Info entry's key value will be written by the fixTLVSectionByName
59 // pass, so create mutable content.
60 auto &TLSInfoEntry = G.createMutableContentBlock(
61 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()),
62 orc::ExecutorAddr(), 8, 0);
63 TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0);
64 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false);
65 }
66
67 private:
getTLSInfoSection(LinkGraph & G)68 Section &getTLSInfoSection(LinkGraph &G) {
69 if (!TLSInfoTable)
70 TLSInfoTable =
71 &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read);
72 return *TLSInfoTable;
73 }
74
getTLSInfoEntryContent() const75 ArrayRef<char> getTLSInfoEntryContent() const {
76 return {reinterpret_cast<const char *>(TLSInfoEntryContent),
77 sizeof(TLSInfoEntryContent)};
78 }
79
80 Section *TLSInfoTable = nullptr;
81 };
82
83 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = {
84 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */
85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/
86 };
87
buildTables_ELF_x86_64(LinkGraph & G)88 Error buildTables_ELF_x86_64(LinkGraph &G) {
89 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n");
90
91 x86_64::GOTTableManager GOT(G);
92 x86_64::PLTTableManager PLT(G, GOT);
93 TLSInfoTableManager_ELF_x86_64 TLSInfo;
94 visitExistingEdges(G, GOT, PLT, TLSInfo);
95 return Error::success();
96 }
97 } // namespace
98
99 namespace llvm {
100 namespace jitlink {
101
102 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> {
103 private:
104 using ELFT = object::ELF64LE;
105
addRelocations()106 Error addRelocations() override {
107 LLVM_DEBUG(dbgs() << "Processing relocations:\n");
108
109 using Base = ELFLinkGraphBuilder<ELFT>;
110 using Self = ELFLinkGraphBuilder_x86_64;
111 for (const auto &RelSect : Base::Sections) {
112 // Validate the section to read relocation entries from.
113 if (RelSect.sh_type == ELF::SHT_REL)
114 return make_error<StringError>(
115 "No SHT_REL in valid x64 ELF object files",
116 inconvertibleErrorCode());
117
118 if (Error Err = Base::forEachRelaRelocation(RelSect, this,
119 &Self::addSingleRelocation))
120 return Err;
121 }
122
123 return Error::success();
124 }
125
addSingleRelocation(const typename ELFT::Rela & Rel,const typename ELFT::Shdr & FixupSection,Block & BlockToFix)126 Error addSingleRelocation(const typename ELFT::Rela &Rel,
127 const typename ELFT::Shdr &FixupSection,
128 Block &BlockToFix) {
129 using Base = ELFLinkGraphBuilder<ELFT>;
130
131 auto ELFReloc = Rel.getType(false);
132
133 // R_X86_64_NONE is a no-op.
134 if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE))
135 return Error::success();
136
137 uint32_t SymbolIndex = Rel.getSymbol(false);
138 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec);
139 if (!ObjSymbol)
140 return ObjSymbol.takeError();
141
142 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex);
143 if (!GraphSymbol)
144 return make_error<StringError>(
145 formatv("Could not find symbol at given index, did you add it to "
146 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}",
147 SymbolIndex, (*ObjSymbol)->st_shndx,
148 Base::GraphSymbols.size()),
149 inconvertibleErrorCode());
150
151 // Validate the relocation kind.
152 int64_t Addend = Rel.r_addend;
153 Edge::Kind Kind = Edge::Invalid;
154
155 switch (ELFReloc) {
156 case ELF::R_X86_64_PC8:
157 Kind = x86_64::Delta8;
158 break;
159 case ELF::R_X86_64_PC16:
160 Kind = x86_64::Delta16;
161 break;
162 case ELF::R_X86_64_PC32:
163 case ELF::R_X86_64_GOTPC32:
164 Kind = x86_64::Delta32;
165 break;
166 case ELF::R_X86_64_PC64:
167 case ELF::R_X86_64_GOTPC64:
168 Kind = x86_64::Delta64;
169 break;
170 case ELF::R_X86_64_32:
171 Kind = x86_64::Pointer32;
172 break;
173 case ELF::R_X86_64_16:
174 Kind = x86_64::Pointer16;
175 break;
176 case ELF::R_X86_64_8:
177 Kind = x86_64::Pointer8;
178 break;
179 case ELF::R_X86_64_32S:
180 Kind = x86_64::Pointer32Signed;
181 break;
182 case ELF::R_X86_64_64:
183 Kind = x86_64::Pointer64;
184 break;
185 case ELF::R_X86_64_SIZE32:
186 Kind = x86_64::Size32;
187 break;
188 case ELF::R_X86_64_SIZE64:
189 Kind = x86_64::Size64;
190 break;
191 case ELF::R_X86_64_GOTPCREL:
192 Kind = x86_64::RequestGOTAndTransformToDelta32;
193 break;
194 case ELF::R_X86_64_REX_GOTPCRELX:
195 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable;
196 Addend = 0;
197 break;
198 case ELF::R_X86_64_TLSGD:
199 Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32;
200 break;
201 case ELF::R_X86_64_GOTPCRELX:
202 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable;
203 Addend = 0;
204 break;
205 case ELF::R_X86_64_GOTPCREL64:
206 Kind = x86_64::RequestGOTAndTransformToDelta64;
207 break;
208 case ELF::R_X86_64_GOT64:
209 Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT;
210 break;
211 case ELF::R_X86_64_GOTOFF64:
212 Kind = x86_64::Delta64FromGOT;
213 break;
214 case ELF::R_X86_64_PLT32:
215 Kind = x86_64::BranchPCRel32;
216 // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to
217 // adjust the addend by '+4' to compensate.
218 Addend += 4;
219 break;
220 default:
221 return make_error<JITLinkError>(
222 "In " + G->getName() + ": Unsupported x86-64 relocation type " +
223 object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc));
224 }
225
226 auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset;
227 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress();
228 Edge GE(Kind, Offset, *GraphSymbol, Addend);
229 LLVM_DEBUG({
230 dbgs() << " ";
231 printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind));
232 dbgs() << "\n";
233 });
234
235 BlockToFix.addEdge(std::move(GE));
236 return Error::success();
237 }
238
239 public:
ELFLinkGraphBuilder_x86_64(StringRef FileName,std::shared_ptr<orc::SymbolStringPool> SSP,const object::ELFFile<object::ELF64LE> & Obj,SubtargetFeatures Features)240 ELFLinkGraphBuilder_x86_64(StringRef FileName,
241 std::shared_ptr<orc::SymbolStringPool> SSP,
242 const object::ELFFile<object::ELF64LE> &Obj,
243 SubtargetFeatures Features)
244 : ELFLinkGraphBuilder(Obj, std::move(SSP), Triple("x86_64-unknown-linux"),
245 std::move(Features), FileName,
246 x86_64::getEdgeKindName) {}
247 };
248
249 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
250 friend class JITLinker<ELFJITLinker_x86_64>;
251
252 public:
ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,std::unique_ptr<LinkGraph> G,PassConfiguration PassConfig)253 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
254 std::unique_ptr<LinkGraph> G,
255 PassConfiguration PassConfig)
256 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {
257
258 if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple()))
259 getPassConfig().PostAllocationPasses.push_back(
260 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); });
261 }
262
263 private:
264 Symbol *GOTSymbol = nullptr;
getOrCreateGOTSymbol(LinkGraph & G)265 Error getOrCreateGOTSymbol(LinkGraph &G) {
266 auto DefineExternalGOTSymbolIfPresent =
267 createDefineExternalSectionStartAndEndSymbolsPass(
268 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc {
269 if (Sym.getName() != nullptr &&
270 *Sym.getName() == ELFGOTSymbolName)
271 if (auto *GOTSection = G.findSectionByName(
272 x86_64::GOTTableManager::getSectionName())) {
273 GOTSymbol = &Sym;
274 return {*GOTSection, true};
275 }
276 return {};
277 });
278
279 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an
280 // external.
281 if (auto Err = DefineExternalGOTSymbolIfPresent(G))
282 return Err;
283
284 // If we succeeded then we're done.
285 if (GOTSymbol)
286 return Error::success();
287
288 // Otherwise look for a GOT section: If it already has a start symbol we'll
289 // record it, otherwise we'll create our own.
290 // If there's a GOT section but we didn't find an external GOT symbol...
291 if (auto *GOTSection =
292 G.findSectionByName(x86_64::GOTTableManager::getSectionName())) {
293
294 // Check for an existing defined symbol.
295 for (auto *Sym : GOTSection->symbols())
296 if (Sym->getName() != nullptr && *Sym->getName() == ELFGOTSymbolName) {
297 GOTSymbol = Sym;
298 return Error::success();
299 }
300
301 // If there's no defined symbol then create one.
302 SectionRange SR(*GOTSection);
303 if (SR.empty())
304 GOTSymbol =
305 &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0,
306 Linkage::Strong, Scope::Local, true);
307 else
308 GOTSymbol =
309 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0,
310 Linkage::Strong, Scope::Local, false, true);
311 }
312
313 // If we still haven't found a GOT symbol then double check the externals.
314 // We may have a GOT-relative reference but no GOT section, in which case
315 // we just need to point the GOT symbol at some address in this graph.
316 if (!GOTSymbol) {
317 for (auto *Sym : G.external_symbols()) {
318 if (*Sym->getName() == ELFGOTSymbolName) {
319 auto Blocks = G.blocks();
320 if (!Blocks.empty()) {
321 G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress());
322 GOTSymbol = Sym;
323 break;
324 }
325 }
326 }
327 }
328
329 return Error::success();
330 }
331
applyFixup(LinkGraph & G,Block & B,const Edge & E) const332 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const {
333 return x86_64::applyFixup(G, B, E, GOTSymbol);
334 }
335 };
336
createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer,std::shared_ptr<orc::SymbolStringPool> SSP)337 Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_x86_64(
338 MemoryBufferRef ObjectBuffer, std::shared_ptr<orc::SymbolStringPool> SSP) {
339 LLVM_DEBUG({
340 dbgs() << "Building jitlink graph for new input "
341 << ObjectBuffer.getBufferIdentifier() << "...\n";
342 });
343
344 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer);
345 if (!ELFObj)
346 return ELFObj.takeError();
347
348 auto Features = (*ELFObj)->getFeatures();
349 if (!Features)
350 return Features.takeError();
351
352 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
353 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), std::move(SSP),
354 ELFObjFile.getELFFile(),
355 std::move(*Features))
356 .buildGraph();
357 }
358
link_ELF_x86_64(std::unique_ptr<LinkGraph> G,std::unique_ptr<JITLinkContext> Ctx)359 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G,
360 std::unique_ptr<JITLinkContext> Ctx) {
361 PassConfiguration Config;
362
363 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) {
364
365 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame"));
366 Config.PrePrunePasses.push_back(EHFrameEdgeFixer(
367 ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64,
368 x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32));
369 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame"));
370
371 // Construct a JITLinker and run the link function.
372 // Add a mark-live pass.
373 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple()))
374 Config.PrePrunePasses.push_back(std::move(MarkLive));
375 else
376 Config.PrePrunePasses.push_back(markAllSymbolsLive);
377
378 // Add an in-place GOT/Stubs/TLSInfoEntry build pass.
379 Config.PostPrunePasses.push_back(buildTables_ELF_x86_64);
380
381 // Resolve any external section start / end symbols.
382 Config.PostAllocationPasses.push_back(
383 createDefineExternalSectionStartAndEndSymbolsPass(
384 identifyELFSectionStartAndEndSymbols));
385
386 // Add GOT/Stubs optimizer pass.
387 Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses);
388 }
389
390 if (auto Err = Ctx->modifyPassConfig(*G, Config))
391 return Ctx->notifyFailed(std::move(Err));
392
393 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config));
394 }
395 } // end namespace jitlink
396 } // end namespace llvm
397