1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/x86-64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" 14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" 15 #include "llvm/ExecutionEngine/JITLink/JITLink.h" 16 #include "llvm/ExecutionEngine/JITLink/TableManager.h" 17 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 18 #include "llvm/Object/ELFObjectFile.h" 19 #include "llvm/Support/Endian.h" 20 21 #include "DefineExternalSectionStartAndEndSymbols.h" 22 #include "EHFrameSupportImpl.h" 23 #include "ELFLinkGraphBuilder.h" 24 #include "JITLinkGeneric.h" 25 26 #define DEBUG_TYPE "jitlink" 27 28 using namespace llvm; 29 using namespace llvm::jitlink; 30 31 namespace { 32 33 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; 34 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO"; 35 36 class TLSInfoTableManager_ELF_x86_64 37 : public TableManager<TLSInfoTableManager_ELF_x86_64> { 38 public: 39 static const uint8_t TLSInfoEntryContent[16]; 40 41 static StringRef getSectionName() { return ELFTLSInfoSectionName; } 42 43 bool visitEdge(LinkGraph &G, Block *B, Edge &E) { 44 if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) { 45 LLVM_DEBUG({ 46 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " 47 << formatv("{0:x}", B->getFixupAddress(E)) << " (" 48 << formatv("{0:x}", B->getAddress()) << " + " 49 << formatv("{0:x}", E.getOffset()) << ")\n"; 50 }); 51 E.setKind(x86_64::Delta32); 52 E.setTarget(getEntryForTarget(G, E.getTarget())); 53 return true; 54 } 55 return false; 56 } 57 58 Symbol &createEntry(LinkGraph &G, Symbol &Target) { 59 // the TLS Info entry's key value will be written by the fixTLVSectionByName 60 // pass, so create mutable content. 61 auto &TLSInfoEntry = G.createMutableContentBlock( 62 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 63 orc::ExecutorAddr(), 8, 0); 64 TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0); 65 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); 66 } 67 68 private: 69 Section &getTLSInfoSection(LinkGraph &G) { 70 if (!TLSInfoTable) 71 TLSInfoTable = 72 &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read); 73 return *TLSInfoTable; 74 } 75 76 ArrayRef<char> getTLSInfoEntryContent() const { 77 return {reinterpret_cast<const char *>(TLSInfoEntryContent), 78 sizeof(TLSInfoEntryContent)}; 79 } 80 81 Section *TLSInfoTable = nullptr; 82 }; 83 84 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = { 85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ 86 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ 87 }; 88 89 Error buildTables_ELF_x86_64(LinkGraph &G) { 90 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); 91 92 x86_64::GOTTableManager GOT; 93 x86_64::PLTTableManager PLT(GOT); 94 TLSInfoTableManager_ELF_x86_64 TLSInfo; 95 visitExistingEdges(G, GOT, PLT, TLSInfo); 96 return Error::success(); 97 } 98 } // namespace 99 100 namespace llvm { 101 namespace jitlink { 102 103 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> { 104 private: 105 using ELFT = object::ELF64LE; 106 107 Error addRelocations() override { 108 LLVM_DEBUG(dbgs() << "Processing relocations:\n"); 109 110 using Base = ELFLinkGraphBuilder<ELFT>; 111 using Self = ELFLinkGraphBuilder_x86_64; 112 for (const auto &RelSect : Base::Sections) { 113 // Validate the section to read relocation entries from. 114 if (RelSect.sh_type == ELF::SHT_REL) 115 return make_error<StringError>( 116 "No SHT_REL in valid x64 ELF object files", 117 inconvertibleErrorCode()); 118 119 if (Error Err = Base::forEachRelaRelocation(RelSect, this, 120 &Self::addSingleRelocation)) 121 return Err; 122 } 123 124 return Error::success(); 125 } 126 127 Error addSingleRelocation(const typename ELFT::Rela &Rel, 128 const typename ELFT::Shdr &FixupSection, 129 Block &BlockToFix) { 130 using Base = ELFLinkGraphBuilder<ELFT>; 131 132 auto ELFReloc = Rel.getType(false); 133 134 // R_X86_64_NONE is a no-op. 135 if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE)) 136 return Error::success(); 137 138 uint32_t SymbolIndex = Rel.getSymbol(false); 139 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); 140 if (!ObjSymbol) 141 return ObjSymbol.takeError(); 142 143 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); 144 if (!GraphSymbol) 145 return make_error<StringError>( 146 formatv("Could not find symbol at given index, did you add it to " 147 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", 148 SymbolIndex, (*ObjSymbol)->st_shndx, 149 Base::GraphSymbols.size()), 150 inconvertibleErrorCode()); 151 152 // Validate the relocation kind. 153 int64_t Addend = Rel.r_addend; 154 Edge::Kind Kind = Edge::Invalid; 155 156 switch (ELFReloc) { 157 case ELF::R_X86_64_PC32: 158 case ELF::R_X86_64_GOTPC32: 159 Kind = x86_64::Delta32; 160 break; 161 case ELF::R_X86_64_PC64: 162 case ELF::R_X86_64_GOTPC64: 163 Kind = x86_64::Delta64; 164 break; 165 case ELF::R_X86_64_32: 166 Kind = x86_64::Pointer32; 167 break; 168 case ELF::R_X86_64_16: 169 Kind = x86_64::Pointer16; 170 break; 171 case ELF::R_X86_64_8: 172 Kind = x86_64::Pointer8; 173 break; 174 case ELF::R_X86_64_32S: 175 Kind = x86_64::Pointer32Signed; 176 break; 177 case ELF::R_X86_64_64: 178 Kind = x86_64::Pointer64; 179 break; 180 case ELF::R_X86_64_GOTPCREL: 181 Kind = x86_64::RequestGOTAndTransformToDelta32; 182 break; 183 case ELF::R_X86_64_REX_GOTPCRELX: 184 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable; 185 Addend = 0; 186 break; 187 case ELF::R_X86_64_TLSGD: 188 Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32; 189 break; 190 case ELF::R_X86_64_GOTPCRELX: 191 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; 192 Addend = 0; 193 break; 194 case ELF::R_X86_64_GOTPCREL64: 195 Kind = x86_64::RequestGOTAndTransformToDelta64; 196 break; 197 case ELF::R_X86_64_GOT64: 198 Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT; 199 break; 200 case ELF::R_X86_64_GOTOFF64: 201 Kind = x86_64::Delta64FromGOT; 202 break; 203 case ELF::R_X86_64_PLT32: 204 Kind = x86_64::BranchPCRel32; 205 // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to 206 // adjust the addend by '+4' to compensate. 207 Addend += 4; 208 break; 209 default: 210 return make_error<JITLinkError>( 211 "In " + G->getName() + ": Unsupported x86-64 relocation type " + 212 object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc)); 213 } 214 215 auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; 216 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); 217 Edge GE(Kind, Offset, *GraphSymbol, Addend); 218 LLVM_DEBUG({ 219 dbgs() << " "; 220 printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind)); 221 dbgs() << "\n"; 222 }); 223 224 BlockToFix.addEdge(std::move(GE)); 225 return Error::success(); 226 } 227 228 public: 229 ELFLinkGraphBuilder_x86_64(StringRef FileName, 230 const object::ELFFile<object::ELF64LE> &Obj, 231 SubtargetFeatures Features) 232 : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), 233 std::move(Features), FileName, 234 x86_64::getEdgeKindName) {} 235 }; 236 237 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { 238 friend class JITLinker<ELFJITLinker_x86_64>; 239 240 public: 241 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, 242 std::unique_ptr<LinkGraph> G, 243 PassConfiguration PassConfig) 244 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { 245 getPassConfig().PostAllocationPasses.push_back( 246 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); 247 } 248 249 private: 250 Symbol *GOTSymbol = nullptr; 251 252 Error getOrCreateGOTSymbol(LinkGraph &G) { 253 auto DefineExternalGOTSymbolIfPresent = 254 createDefineExternalSectionStartAndEndSymbolsPass( 255 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { 256 if (Sym.getName() == ELFGOTSymbolName) 257 if (auto *GOTSection = G.findSectionByName( 258 x86_64::GOTTableManager::getSectionName())) { 259 GOTSymbol = &Sym; 260 return {*GOTSection, true}; 261 } 262 return {}; 263 }); 264 265 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an 266 // external. 267 if (auto Err = DefineExternalGOTSymbolIfPresent(G)) 268 return Err; 269 270 // If we succeeded then we're done. 271 if (GOTSymbol) 272 return Error::success(); 273 274 // Otherwise look for a GOT section: If it already has a start symbol we'll 275 // record it, otherwise we'll create our own. 276 // If there's a GOT section but we didn't find an external GOT symbol... 277 if (auto *GOTSection = 278 G.findSectionByName(x86_64::GOTTableManager::getSectionName())) { 279 280 // Check for an existing defined symbol. 281 for (auto *Sym : GOTSection->symbols()) 282 if (Sym->getName() == ELFGOTSymbolName) { 283 GOTSymbol = Sym; 284 return Error::success(); 285 } 286 287 // If there's no defined symbol then create one. 288 SectionRange SR(*GOTSection); 289 if (SR.empty()) 290 GOTSymbol = 291 &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0, 292 Linkage::Strong, Scope::Local, true); 293 else 294 GOTSymbol = 295 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, 296 Linkage::Strong, Scope::Local, false, true); 297 } 298 299 // If we still haven't found a GOT symbol then double check the externals. 300 // We may have a GOT-relative reference but no GOT section, in which case 301 // we just need to point the GOT symbol at some address in this graph. 302 if (!GOTSymbol) { 303 for (auto *Sym : G.external_symbols()) { 304 if (Sym->getName() == ELFGOTSymbolName) { 305 auto Blocks = G.blocks(); 306 if (!Blocks.empty()) { 307 G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress()); 308 GOTSymbol = Sym; 309 break; 310 } 311 } 312 } 313 } 314 315 return Error::success(); 316 } 317 318 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { 319 return x86_64::applyFixup(G, B, E, GOTSymbol); 320 } 321 }; 322 323 Expected<std::unique_ptr<LinkGraph>> 324 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { 325 LLVM_DEBUG({ 326 dbgs() << "Building jitlink graph for new input " 327 << ObjectBuffer.getBufferIdentifier() << "...\n"; 328 }); 329 330 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 331 if (!ELFObj) 332 return ELFObj.takeError(); 333 334 auto Features = (*ELFObj)->getFeatures(); 335 if (!Features) 336 return Features.takeError(); 337 338 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 339 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), 340 ELFObjFile.getELFFile(), 341 std::move(*Features)) 342 .buildGraph(); 343 } 344 345 static SectionRangeSymbolDesc 346 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { 347 constexpr StringRef StartSymbolPrefix = "__start"; 348 constexpr StringRef EndSymbolPrefix = "__end"; 349 350 auto SymName = Sym.getName(); 351 if (SymName.startswith(StartSymbolPrefix)) { 352 if (auto *Sec = 353 G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) 354 return {*Sec, true}; 355 } else if (SymName.startswith(EndSymbolPrefix)) { 356 if (auto *Sec = 357 G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) 358 return {*Sec, false}; 359 } 360 return {}; 361 } 362 363 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, 364 std::unique_ptr<JITLinkContext> Ctx) { 365 PassConfiguration Config; 366 367 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { 368 369 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); 370 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 371 ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64, 372 x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32)); 373 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); 374 375 // Construct a JITLinker and run the link function. 376 // Add a mark-live pass. 377 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) 378 Config.PrePrunePasses.push_back(std::move(MarkLive)); 379 else 380 Config.PrePrunePasses.push_back(markAllSymbolsLive); 381 382 // Add an in-place GOT/Stubs/TLSInfoEntry build pass. 383 Config.PostPrunePasses.push_back(buildTables_ELF_x86_64); 384 385 // Resolve any external section start / end symbols. 386 Config.PostAllocationPasses.push_back( 387 createDefineExternalSectionStartAndEndSymbolsPass( 388 identifyELFSectionStartAndEndSymbols)); 389 390 // Add GOT/Stubs optimizer pass. 391 Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses); 392 } 393 394 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 395 return Ctx->notifyFailed(std::move(Err)); 396 397 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); 398 } 399 } // end namespace jitlink 400 } // end namespace llvm 401