1 //===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/x86-64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" 14 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" 15 #include "llvm/ExecutionEngine/JITLink/JITLink.h" 16 #include "llvm/ExecutionEngine/JITLink/TableManager.h" 17 #include "llvm/ExecutionEngine/JITLink/x86_64.h" 18 #include "llvm/Object/ELFObjectFile.h" 19 20 #include "DefineExternalSectionStartAndEndSymbols.h" 21 #include "EHFrameSupportImpl.h" 22 #include "ELFLinkGraphBuilder.h" 23 #include "JITLinkGeneric.h" 24 25 #define DEBUG_TYPE "jitlink" 26 27 using namespace llvm; 28 using namespace llvm::jitlink; 29 30 namespace { 31 32 constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; 33 constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO"; 34 35 class TLSInfoTableManager_ELF_x86_64 36 : public TableManager<TLSInfoTableManager_ELF_x86_64> { 37 public: 38 static const uint8_t TLSInfoEntryContent[16]; 39 40 static StringRef getSectionName() { return ELFTLSInfoSectionName; } 41 42 bool visitEdge(LinkGraph &G, Block *B, Edge &E) { 43 if (E.getKind() == x86_64::RequestTLSDescInGOTAndTransformToDelta32) { 44 LLVM_DEBUG({ 45 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " 46 << formatv("{0:x}", B->getFixupAddress(E)) << " (" 47 << formatv("{0:x}", B->getAddress()) << " + " 48 << formatv("{0:x}", E.getOffset()) << ")\n"; 49 }); 50 E.setKind(x86_64::Delta32); 51 E.setTarget(getEntryForTarget(G, E.getTarget())); 52 return true; 53 } 54 return false; 55 } 56 57 Symbol &createEntry(LinkGraph &G, Symbol &Target) { 58 // the TLS Info entry's key value will be written by the fixTLVSectionByName 59 // pass, so create mutable content. 60 auto &TLSInfoEntry = G.createMutableContentBlock( 61 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 62 orc::ExecutorAddr(), 8, 0); 63 TLSInfoEntry.addEdge(x86_64::Pointer64, 8, Target, 0); 64 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); 65 } 66 67 private: 68 Section &getTLSInfoSection(LinkGraph &G) { 69 if (!TLSInfoTable) 70 TLSInfoTable = 71 &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read); 72 return *TLSInfoTable; 73 } 74 75 ArrayRef<char> getTLSInfoEntryContent() const { 76 return {reinterpret_cast<const char *>(TLSInfoEntryContent), 77 sizeof(TLSInfoEntryContent)}; 78 } 79 80 Section *TLSInfoTable = nullptr; 81 }; 82 83 const uint8_t TLSInfoTableManager_ELF_x86_64::TLSInfoEntryContent[16] = { 84 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ 85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ 86 }; 87 88 Error buildTables_ELF_x86_64(LinkGraph &G) { 89 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); 90 91 x86_64::GOTTableManager GOT; 92 x86_64::PLTTableManager PLT(GOT); 93 TLSInfoTableManager_ELF_x86_64 TLSInfo; 94 visitExistingEdges(G, GOT, PLT, TLSInfo); 95 return Error::success(); 96 } 97 } // namespace 98 99 namespace llvm { 100 namespace jitlink { 101 102 class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> { 103 private: 104 using ELFT = object::ELF64LE; 105 106 Error addRelocations() override { 107 LLVM_DEBUG(dbgs() << "Processing relocations:\n"); 108 109 using Base = ELFLinkGraphBuilder<ELFT>; 110 using Self = ELFLinkGraphBuilder_x86_64; 111 for (const auto &RelSect : Base::Sections) { 112 // Validate the section to read relocation entries from. 113 if (RelSect.sh_type == ELF::SHT_REL) 114 return make_error<StringError>( 115 "No SHT_REL in valid x64 ELF object files", 116 inconvertibleErrorCode()); 117 118 if (Error Err = Base::forEachRelaRelocation(RelSect, this, 119 &Self::addSingleRelocation)) 120 return Err; 121 } 122 123 return Error::success(); 124 } 125 126 Error addSingleRelocation(const typename ELFT::Rela &Rel, 127 const typename ELFT::Shdr &FixupSection, 128 Block &BlockToFix) { 129 using Base = ELFLinkGraphBuilder<ELFT>; 130 131 auto ELFReloc = Rel.getType(false); 132 133 // R_X86_64_NONE is a no-op. 134 if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE)) 135 return Error::success(); 136 137 uint32_t SymbolIndex = Rel.getSymbol(false); 138 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); 139 if (!ObjSymbol) 140 return ObjSymbol.takeError(); 141 142 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); 143 if (!GraphSymbol) 144 return make_error<StringError>( 145 formatv("Could not find symbol at given index, did you add it to " 146 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", 147 SymbolIndex, (*ObjSymbol)->st_shndx, 148 Base::GraphSymbols.size()), 149 inconvertibleErrorCode()); 150 151 // Validate the relocation kind. 152 int64_t Addend = Rel.r_addend; 153 Edge::Kind Kind = Edge::Invalid; 154 155 switch (ELFReloc) { 156 case ELF::R_X86_64_PC32: 157 case ELF::R_X86_64_GOTPC32: 158 Kind = x86_64::Delta32; 159 break; 160 case ELF::R_X86_64_PC64: 161 case ELF::R_X86_64_GOTPC64: 162 Kind = x86_64::Delta64; 163 break; 164 case ELF::R_X86_64_32: 165 Kind = x86_64::Pointer32; 166 break; 167 case ELF::R_X86_64_16: 168 Kind = x86_64::Pointer16; 169 break; 170 case ELF::R_X86_64_8: 171 Kind = x86_64::Pointer8; 172 break; 173 case ELF::R_X86_64_32S: 174 Kind = x86_64::Pointer32Signed; 175 break; 176 case ELF::R_X86_64_64: 177 Kind = x86_64::Pointer64; 178 break; 179 case ELF::R_X86_64_GOTPCREL: 180 Kind = x86_64::RequestGOTAndTransformToDelta32; 181 break; 182 case ELF::R_X86_64_REX_GOTPCRELX: 183 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable; 184 Addend = 0; 185 break; 186 case ELF::R_X86_64_TLSGD: 187 Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32; 188 break; 189 case ELF::R_X86_64_GOTPCRELX: 190 Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; 191 Addend = 0; 192 break; 193 case ELF::R_X86_64_GOTPCREL64: 194 Kind = x86_64::RequestGOTAndTransformToDelta64; 195 break; 196 case ELF::R_X86_64_GOT64: 197 Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT; 198 break; 199 case ELF::R_X86_64_GOTOFF64: 200 Kind = x86_64::Delta64FromGOT; 201 break; 202 case ELF::R_X86_64_PLT32: 203 Kind = x86_64::BranchPCRel32; 204 // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to 205 // adjust the addend by '+4' to compensate. 206 Addend += 4; 207 break; 208 default: 209 return make_error<JITLinkError>( 210 "In " + G->getName() + ": Unsupported x86-64 relocation type " + 211 object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc)); 212 } 213 214 auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; 215 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); 216 Edge GE(Kind, Offset, *GraphSymbol, Addend); 217 LLVM_DEBUG({ 218 dbgs() << " "; 219 printEdge(dbgs(), BlockToFix, GE, x86_64::getEdgeKindName(Kind)); 220 dbgs() << "\n"; 221 }); 222 223 BlockToFix.addEdge(std::move(GE)); 224 return Error::success(); 225 } 226 227 public: 228 ELFLinkGraphBuilder_x86_64(StringRef FileName, 229 const object::ELFFile<object::ELF64LE> &Obj, 230 SubtargetFeatures Features) 231 : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), 232 std::move(Features), FileName, 233 x86_64::getEdgeKindName) {} 234 }; 235 236 class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { 237 friend class JITLinker<ELFJITLinker_x86_64>; 238 239 public: 240 ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, 241 std::unique_ptr<LinkGraph> G, 242 PassConfiguration PassConfig) 243 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { 244 245 if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple())) 246 getPassConfig().PostAllocationPasses.push_back( 247 [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); 248 } 249 250 private: 251 Symbol *GOTSymbol = nullptr; 252 253 Error getOrCreateGOTSymbol(LinkGraph &G) { 254 auto DefineExternalGOTSymbolIfPresent = 255 createDefineExternalSectionStartAndEndSymbolsPass( 256 [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { 257 if (Sym.getName() == ELFGOTSymbolName) 258 if (auto *GOTSection = G.findSectionByName( 259 x86_64::GOTTableManager::getSectionName())) { 260 GOTSymbol = &Sym; 261 return {*GOTSection, true}; 262 } 263 return {}; 264 }); 265 266 // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an 267 // external. 268 if (auto Err = DefineExternalGOTSymbolIfPresent(G)) 269 return Err; 270 271 // If we succeeded then we're done. 272 if (GOTSymbol) 273 return Error::success(); 274 275 // Otherwise look for a GOT section: If it already has a start symbol we'll 276 // record it, otherwise we'll create our own. 277 // If there's a GOT section but we didn't find an external GOT symbol... 278 if (auto *GOTSection = 279 G.findSectionByName(x86_64::GOTTableManager::getSectionName())) { 280 281 // Check for an existing defined symbol. 282 for (auto *Sym : GOTSection->symbols()) 283 if (Sym->getName() == ELFGOTSymbolName) { 284 GOTSymbol = Sym; 285 return Error::success(); 286 } 287 288 // If there's no defined symbol then create one. 289 SectionRange SR(*GOTSection); 290 if (SR.empty()) 291 GOTSymbol = 292 &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0, 293 Linkage::Strong, Scope::Local, true); 294 else 295 GOTSymbol = 296 &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, 297 Linkage::Strong, Scope::Local, false, true); 298 } 299 300 // If we still haven't found a GOT symbol then double check the externals. 301 // We may have a GOT-relative reference but no GOT section, in which case 302 // we just need to point the GOT symbol at some address in this graph. 303 if (!GOTSymbol) { 304 for (auto *Sym : G.external_symbols()) { 305 if (Sym->getName() == ELFGOTSymbolName) { 306 auto Blocks = G.blocks(); 307 if (!Blocks.empty()) { 308 G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress()); 309 GOTSymbol = Sym; 310 break; 311 } 312 } 313 } 314 } 315 316 return Error::success(); 317 } 318 319 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { 320 return x86_64::applyFixup(G, B, E, GOTSymbol); 321 } 322 }; 323 324 Expected<std::unique_ptr<LinkGraph>> 325 createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { 326 LLVM_DEBUG({ 327 dbgs() << "Building jitlink graph for new input " 328 << ObjectBuffer.getBufferIdentifier() << "...\n"; 329 }); 330 331 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 332 if (!ELFObj) 333 return ELFObj.takeError(); 334 335 auto Features = (*ELFObj)->getFeatures(); 336 if (!Features) 337 return Features.takeError(); 338 339 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 340 return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), 341 ELFObjFile.getELFFile(), 342 std::move(*Features)) 343 .buildGraph(); 344 } 345 346 static SectionRangeSymbolDesc 347 identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { 348 constexpr StringRef StartSymbolPrefix = "__start"; 349 constexpr StringRef EndSymbolPrefix = "__end"; 350 351 auto SymName = Sym.getName(); 352 if (SymName.starts_with(StartSymbolPrefix)) { 353 if (auto *Sec = 354 G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) 355 return {*Sec, true}; 356 } else if (SymName.starts_with(EndSymbolPrefix)) { 357 if (auto *Sec = 358 G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) 359 return {*Sec, false}; 360 } 361 return {}; 362 } 363 364 void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, 365 std::unique_ptr<JITLinkContext> Ctx) { 366 PassConfiguration Config; 367 368 if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { 369 370 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); 371 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 372 ".eh_frame", x86_64::PointerSize, x86_64::Pointer32, x86_64::Pointer64, 373 x86_64::Delta32, x86_64::Delta64, x86_64::NegDelta32)); 374 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); 375 376 // Construct a JITLinker and run the link function. 377 // Add a mark-live pass. 378 if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) 379 Config.PrePrunePasses.push_back(std::move(MarkLive)); 380 else 381 Config.PrePrunePasses.push_back(markAllSymbolsLive); 382 383 // Add an in-place GOT/Stubs/TLSInfoEntry build pass. 384 Config.PostPrunePasses.push_back(buildTables_ELF_x86_64); 385 386 // Resolve any external section start / end symbols. 387 Config.PostAllocationPasses.push_back( 388 createDefineExternalSectionStartAndEndSymbolsPass( 389 identifyELFSectionStartAndEndSymbols)); 390 391 // Add GOT/Stubs optimizer pass. 392 Config.PreFixupPasses.push_back(x86_64::optimizeGOTAndStubAccesses); 393 } 394 395 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 396 return Ctx->notifyFailed(std::move(Err)); 397 398 ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); 399 } 400 } // end namespace jitlink 401 } // end namespace llvm 402