1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/aarch64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" 14 #include "EHFrameSupportImpl.h" 15 #include "ELFLinkGraphBuilder.h" 16 #include "JITLinkGeneric.h" 17 #include "llvm/BinaryFormat/ELF.h" 18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" 19 #include "llvm/ExecutionEngine/JITLink/aarch64.h" 20 #include "llvm/Object/ELFObjectFile.h" 21 #include "llvm/Support/Endian.h" 22 23 #define DEBUG_TYPE "jitlink" 24 25 using namespace llvm; 26 using namespace llvm::jitlink; 27 28 namespace { 29 30 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> { 31 friend class JITLinker<ELFJITLinker_aarch64>; 32 33 public: 34 ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx, 35 std::unique_ptr<LinkGraph> G, 36 PassConfiguration PassConfig) 37 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} 38 39 private: 40 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { 41 return aarch64::applyFixup(G, B, E); 42 } 43 }; 44 45 template <typename ELFT> 46 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> { 47 private: 48 enum ELFAArch64RelocationKind : Edge::Kind { 49 ELFCall26 = Edge::FirstRelocation, 50 ELFAdrPage21, 51 ELFAddAbs12, 52 ELFLdSt8Abs12, 53 ELFLdSt16Abs12, 54 ELFLdSt32Abs12, 55 ELFLdSt64Abs12, 56 ELFLdSt128Abs12, 57 ELFMovwAbsG0, 58 ELFMovwAbsG1, 59 ELFMovwAbsG2, 60 ELFMovwAbsG3, 61 ELFAbs64, 62 ELFPrel32, 63 ELFPrel64, 64 ELFAdrGOTPage21, 65 ELFLd64GOTLo12, 66 ELFTLSDescAdrPage21, 67 ELFTLSDescAddLo12, 68 ELFTLSDescLd64Lo12, 69 ELFTLSDescCall, 70 }; 71 72 static Expected<ELFAArch64RelocationKind> 73 getRelocationKind(const uint32_t Type) { 74 using namespace aarch64; 75 switch (Type) { 76 case ELF::R_AARCH64_CALL26: 77 case ELF::R_AARCH64_JUMP26: 78 return ELFCall26; 79 case ELF::R_AARCH64_ADR_PREL_PG_HI21: 80 return ELFAdrPage21; 81 case ELF::R_AARCH64_ADD_ABS_LO12_NC: 82 return ELFAddAbs12; 83 case ELF::R_AARCH64_LDST8_ABS_LO12_NC: 84 return ELFLdSt8Abs12; 85 case ELF::R_AARCH64_LDST16_ABS_LO12_NC: 86 return ELFLdSt16Abs12; 87 case ELF::R_AARCH64_LDST32_ABS_LO12_NC: 88 return ELFLdSt32Abs12; 89 case ELF::R_AARCH64_LDST64_ABS_LO12_NC: 90 return ELFLdSt64Abs12; 91 case ELF::R_AARCH64_LDST128_ABS_LO12_NC: 92 return ELFLdSt128Abs12; 93 case ELF::R_AARCH64_MOVW_UABS_G0_NC: 94 return ELFMovwAbsG0; 95 case ELF::R_AARCH64_MOVW_UABS_G1_NC: 96 return ELFMovwAbsG1; 97 case ELF::R_AARCH64_MOVW_UABS_G2_NC: 98 return ELFMovwAbsG2; 99 case ELF::R_AARCH64_MOVW_UABS_G3: 100 return ELFMovwAbsG3; 101 case ELF::R_AARCH64_ABS64: 102 return ELFAbs64; 103 case ELF::R_AARCH64_PREL32: 104 return ELFPrel32; 105 case ELF::R_AARCH64_PREL64: 106 return ELFPrel64; 107 case ELF::R_AARCH64_ADR_GOT_PAGE: 108 return ELFAdrGOTPage21; 109 case ELF::R_AARCH64_LD64_GOT_LO12_NC: 110 return ELFLd64GOTLo12; 111 case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: 112 return ELFTLSDescAdrPage21; 113 case ELF::R_AARCH64_TLSDESC_ADD_LO12: 114 return ELFTLSDescAddLo12; 115 case ELF::R_AARCH64_TLSDESC_LD64_LO12: 116 return ELFTLSDescLd64Lo12; 117 case ELF::R_AARCH64_TLSDESC_CALL: 118 return ELFTLSDescCall; 119 } 120 121 return make_error<JITLinkError>( 122 "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) + 123 object::getELFRelocationTypeName(ELF::EM_AARCH64, Type)); 124 } 125 126 Error addRelocations() override { 127 LLVM_DEBUG(dbgs() << "Processing relocations:\n"); 128 129 using Base = ELFLinkGraphBuilder<ELFT>; 130 using Self = ELFLinkGraphBuilder_aarch64<ELFT>; 131 for (const auto &RelSect : Base::Sections) 132 if (Error Err = Base::forEachRelocation(RelSect, this, 133 &Self::addSingleRelocation)) 134 return Err; 135 136 return Error::success(); 137 } 138 139 Error addSingleRelocation(const typename ELFT::Rela &Rel, 140 const typename ELFT::Shdr &FixupSect, 141 Block &BlockToFix) { 142 using support::ulittle32_t; 143 using Base = ELFLinkGraphBuilder<ELFT>; 144 145 uint32_t SymbolIndex = Rel.getSymbol(false); 146 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); 147 if (!ObjSymbol) 148 return ObjSymbol.takeError(); 149 150 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); 151 if (!GraphSymbol) 152 return make_error<StringError>( 153 formatv("Could not find symbol at given index, did you add it to " 154 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", 155 SymbolIndex, (*ObjSymbol)->st_shndx, 156 Base::GraphSymbols.size()), 157 inconvertibleErrorCode()); 158 159 uint32_t Type = Rel.getType(false); 160 Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type); 161 if (!RelocKind) 162 return RelocKind.takeError(); 163 164 int64_t Addend = Rel.r_addend; 165 orc::ExecutorAddr FixupAddress = 166 orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; 167 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); 168 169 // Get a pointer to the fixup content. 170 const void *FixupContent = BlockToFix.getContent().data() + 171 (FixupAddress - BlockToFix.getAddress()); 172 173 Edge::Kind Kind = Edge::Invalid; 174 175 switch (*RelocKind) { 176 case ELFCall26: { 177 Kind = aarch64::Branch26; 178 break; 179 } 180 case ELFAdrPage21: { 181 Kind = aarch64::Page21; 182 break; 183 } 184 case ELFAddAbs12: { 185 Kind = aarch64::PageOffset12; 186 break; 187 } 188 case ELFLdSt8Abs12: { 189 uint32_t Instr = *(const ulittle32_t *)FixupContent; 190 if (!aarch64::isLoadStoreImm12(Instr) || 191 aarch64::getPageOffset12Shift(Instr) != 0) 192 return make_error<JITLinkError>( 193 "R_AARCH64_LDST8_ABS_LO12_NC target is not a " 194 "LDRB/STRB (imm12) instruction"); 195 196 Kind = aarch64::PageOffset12; 197 break; 198 } 199 case ELFLdSt16Abs12: { 200 uint32_t Instr = *(const ulittle32_t *)FixupContent; 201 if (!aarch64::isLoadStoreImm12(Instr) || 202 aarch64::getPageOffset12Shift(Instr) != 1) 203 return make_error<JITLinkError>( 204 "R_AARCH64_LDST16_ABS_LO12_NC target is not a " 205 "LDRH/STRH (imm12) instruction"); 206 207 Kind = aarch64::PageOffset12; 208 break; 209 } 210 case ELFLdSt32Abs12: { 211 uint32_t Instr = *(const ulittle32_t *)FixupContent; 212 if (!aarch64::isLoadStoreImm12(Instr) || 213 aarch64::getPageOffset12Shift(Instr) != 2) 214 return make_error<JITLinkError>( 215 "R_AARCH64_LDST32_ABS_LO12_NC target is not a " 216 "LDR/STR (imm12, 32 bit) instruction"); 217 218 Kind = aarch64::PageOffset12; 219 break; 220 } 221 case ELFLdSt64Abs12: { 222 uint32_t Instr = *(const ulittle32_t *)FixupContent; 223 if (!aarch64::isLoadStoreImm12(Instr) || 224 aarch64::getPageOffset12Shift(Instr) != 3) 225 return make_error<JITLinkError>( 226 "R_AARCH64_LDST64_ABS_LO12_NC target is not a " 227 "LDR/STR (imm12, 64 bit) instruction"); 228 229 Kind = aarch64::PageOffset12; 230 break; 231 } 232 case ELFLdSt128Abs12: { 233 uint32_t Instr = *(const ulittle32_t *)FixupContent; 234 if (!aarch64::isLoadStoreImm12(Instr) || 235 aarch64::getPageOffset12Shift(Instr) != 4) 236 return make_error<JITLinkError>( 237 "R_AARCH64_LDST128_ABS_LO12_NC target is not a " 238 "LDR/STR (imm12, 128 bit) instruction"); 239 240 Kind = aarch64::PageOffset12; 241 break; 242 } 243 case ELFMovwAbsG0: { 244 uint32_t Instr = *(const ulittle32_t *)FixupContent; 245 if (!aarch64::isMoveWideImm16(Instr) || 246 aarch64::getMoveWide16Shift(Instr) != 0) 247 return make_error<JITLinkError>( 248 "R_AARCH64_MOVW_UABS_G0_NC target is not a " 249 "MOVK/MOVZ (imm16, LSL #0) instruction"); 250 251 Kind = aarch64::MoveWide16; 252 break; 253 } 254 case ELFMovwAbsG1: { 255 uint32_t Instr = *(const ulittle32_t *)FixupContent; 256 if (!aarch64::isMoveWideImm16(Instr) || 257 aarch64::getMoveWide16Shift(Instr) != 16) 258 return make_error<JITLinkError>( 259 "R_AARCH64_MOVW_UABS_G1_NC target is not a " 260 "MOVK/MOVZ (imm16, LSL #16) instruction"); 261 262 Kind = aarch64::MoveWide16; 263 break; 264 } 265 case ELFMovwAbsG2: { 266 uint32_t Instr = *(const ulittle32_t *)FixupContent; 267 if (!aarch64::isMoveWideImm16(Instr) || 268 aarch64::getMoveWide16Shift(Instr) != 32) 269 return make_error<JITLinkError>( 270 "R_AARCH64_MOVW_UABS_G2_NC target is not a " 271 "MOVK/MOVZ (imm16, LSL #32) instruction"); 272 273 Kind = aarch64::MoveWide16; 274 break; 275 } 276 case ELFMovwAbsG3: { 277 uint32_t Instr = *(const ulittle32_t *)FixupContent; 278 if (!aarch64::isMoveWideImm16(Instr) || 279 aarch64::getMoveWide16Shift(Instr) != 48) 280 return make_error<JITLinkError>( 281 "R_AARCH64_MOVW_UABS_G3 target is not a " 282 "MOVK/MOVZ (imm16, LSL #48) instruction"); 283 284 Kind = aarch64::MoveWide16; 285 break; 286 } 287 case ELFAbs64: { 288 Kind = aarch64::Pointer64; 289 break; 290 } 291 case ELFPrel32: { 292 Kind = aarch64::Delta32; 293 break; 294 } 295 case ELFPrel64: { 296 Kind = aarch64::Delta64; 297 break; 298 } 299 case ELFAdrGOTPage21: { 300 Kind = aarch64::GOTPage21; 301 break; 302 } 303 case ELFLd64GOTLo12: { 304 Kind = aarch64::GOTPageOffset12; 305 break; 306 } 307 case ELFTLSDescAdrPage21: { 308 Kind = aarch64::TLSDescPage21; 309 break; 310 } 311 case ELFTLSDescAddLo12: { 312 Kind = aarch64::TLSDescPageOffset12; 313 break; 314 } 315 case ELFTLSDescLd64Lo12: { 316 Kind = aarch64::TLSDescPageOffset12; 317 break; 318 } 319 case ELFTLSDescCall: { 320 return Error::success(); 321 } 322 }; 323 324 Edge GE(Kind, Offset, *GraphSymbol, Addend); 325 LLVM_DEBUG({ 326 dbgs() << " "; 327 printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind)); 328 dbgs() << "\n"; 329 }); 330 331 BlockToFix.addEdge(std::move(GE)); 332 333 return Error::success(); 334 } 335 336 /// Return the string name of the given ELF aarch64 edge kind. 337 const char *getELFAArch64RelocationKindName(Edge::Kind R) { 338 switch (R) { 339 case ELFCall26: 340 return "ELFCall26"; 341 case ELFAdrPage21: 342 return "ELFAdrPage21"; 343 case ELFAddAbs12: 344 return "ELFAddAbs12"; 345 case ELFLdSt8Abs12: 346 return "ELFLdSt8Abs12"; 347 case ELFLdSt16Abs12: 348 return "ELFLdSt16Abs12"; 349 case ELFLdSt32Abs12: 350 return "ELFLdSt32Abs12"; 351 case ELFLdSt64Abs12: 352 return "ELFLdSt64Abs12"; 353 case ELFLdSt128Abs12: 354 return "ELFLdSt128Abs12"; 355 case ELFMovwAbsG0: 356 return "ELFMovwAbsG0"; 357 case ELFMovwAbsG1: 358 return "ELFMovwAbsG1"; 359 case ELFMovwAbsG2: 360 return "ELFMovwAbsG2"; 361 case ELFMovwAbsG3: 362 return "ELFMovwAbsG3"; 363 case ELFAbs64: 364 return "ELFAbs64"; 365 case ELFPrel32: 366 return "ELFPrel32"; 367 case ELFPrel64: 368 return "ELFPrel64"; 369 case ELFAdrGOTPage21: 370 return "ELFAdrGOTPage21"; 371 case ELFLd64GOTLo12: 372 return "ELFLd64GOTLo12"; 373 case ELFTLSDescAdrPage21: 374 return "ELFTLSDescAdrPage21"; 375 case ELFTLSDescAddLo12: 376 return "ELFTLSDescAddLo12"; 377 case ELFTLSDescLd64Lo12: 378 return "ELFTLSDescLd64Lo12"; 379 case ELFTLSDescCall: 380 return "ELFTLSDescCall"; 381 default: 382 return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); 383 } 384 } 385 386 public: 387 ELFLinkGraphBuilder_aarch64(StringRef FileName, 388 const object::ELFFile<ELFT> &Obj, const Triple T) 389 : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, 390 aarch64::getEdgeKindName) {} 391 }; 392 393 // TLS Info Builder. 394 class TLSInfoTableManager_ELF_aarch64 395 : public TableManager<TLSInfoTableManager_ELF_aarch64> { 396 public: 397 static StringRef getSectionName() { return "$__TLSINFO"; } 398 399 static const uint8_t TLSInfoEntryContent[16]; 400 401 bool visitEdge(LinkGraph &G, Block *B, Edge &E) { return false; } 402 403 Symbol &createEntry(LinkGraph &G, Symbol &Target) { 404 // the TLS Info entry's key value will be written by the fixTLVSectionByName 405 // pass, so create mutable content. 406 auto &TLSInfoEntry = G.createMutableContentBlock( 407 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 408 orc::ExecutorAddr(), 8, 0); 409 TLSInfoEntry.addEdge(aarch64::Pointer64, 8, Target, 0); 410 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); 411 } 412 413 private: 414 Section &getTLSInfoSection(LinkGraph &G) { 415 if (!TLSInfoTable) 416 TLSInfoTable = &G.createSection(getSectionName(), MemProt::Read); 417 return *TLSInfoTable; 418 } 419 420 ArrayRef<char> getTLSInfoEntryContent() const { 421 return {reinterpret_cast<const char *>(TLSInfoEntryContent), 422 sizeof(TLSInfoEntryContent)}; 423 } 424 425 Section *TLSInfoTable = nullptr; 426 }; 427 428 const uint8_t TLSInfoTableManager_ELF_aarch64::TLSInfoEntryContent[16] = { 429 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ 430 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ 431 }; 432 433 // TLS Descriptor Builder. 434 class TLSDescTableManager_ELF_aarch64 435 : public TableManager<TLSDescTableManager_ELF_aarch64> { 436 public: 437 TLSDescTableManager_ELF_aarch64( 438 TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager) 439 : TLSInfoTableManager(TLSInfoTableManager) {} 440 441 static StringRef getSectionName() { return "$__TLSDESC"; } 442 443 static const uint8_t TLSDescEntryContent[16]; 444 445 bool visitEdge(LinkGraph &G, Block *B, Edge &E) { 446 Edge::Kind KindToSet = Edge::Invalid; 447 switch (E.getKind()) { 448 case aarch64::TLSDescPage21: { 449 KindToSet = aarch64::Page21; 450 break; 451 } 452 case aarch64::TLSDescPageOffset12: { 453 KindToSet = aarch64::PageOffset12; 454 break; 455 } 456 default: 457 return false; 458 } 459 assert(KindToSet != Edge::Invalid && 460 "Fell through switch, but no new kind to set"); 461 DEBUG_WITH_TYPE("jitlink", { 462 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " 463 << B->getFixupAddress(E) << " (" << B->getAddress() << " + " 464 << formatv("{0:x}", E.getOffset()) << ")\n"; 465 }); 466 E.setKind(KindToSet); 467 E.setTarget(getEntryForTarget(G, E.getTarget())); 468 return true; 469 } 470 471 Symbol &createEntry(LinkGraph &G, Symbol &Target) { 472 auto &EntryBlock = 473 G.createContentBlock(getTLSDescSection(G), getTLSDescBlockContent(), 474 orc::ExecutorAddr(), 8, 0); 475 EntryBlock.addEdge(aarch64::Pointer64, 0, getTLSDescResolver(G), 0); 476 EntryBlock.addEdge(aarch64::Pointer64, 8, 477 TLSInfoTableManager.getEntryForTarget(G, Target), 0); 478 return G.addAnonymousSymbol(EntryBlock, 0, 8, false, false); 479 } 480 481 private: 482 Section &getTLSDescSection(LinkGraph &G) { 483 if (!GOTSection) 484 GOTSection = &G.createSection(getSectionName(), MemProt::Read); 485 return *GOTSection; 486 } 487 488 Symbol &getTLSDescResolver(LinkGraph &G) { 489 if (!TLSDescResolver) 490 TLSDescResolver = 491 &G.addExternalSymbol("__tlsdesc_resolver", 8, Linkage::Strong); 492 return *TLSDescResolver; 493 } 494 495 ArrayRef<char> getTLSDescBlockContent() { 496 return {reinterpret_cast<const char *>(TLSDescEntryContent), 497 sizeof(TLSDescEntryContent)}; 498 } 499 500 Section *GOTSection = nullptr; 501 Symbol *TLSDescResolver = nullptr; 502 TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager; 503 }; 504 505 const uint8_t TLSDescTableManager_ELF_aarch64::TLSDescEntryContent[16] = { 506 0x00, 0x00, 0x00, 0x00, 507 0x00, 0x00, 0x00, 0x00, /*resolver function pointer*/ 508 0x00, 0x00, 0x00, 0x00, 509 0x00, 0x00, 0x00, 0x00 /*pointer to tls info*/ 510 }; 511 512 Error buildTables_ELF_aarch64(LinkGraph &G) { 513 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); 514 515 aarch64::GOTTableManager GOT; 516 aarch64::PLTTableManager PLT(GOT); 517 TLSInfoTableManager_ELF_aarch64 TLSInfo; 518 TLSDescTableManager_ELF_aarch64 TLSDesc(TLSInfo); 519 visitExistingEdges(G, GOT, PLT, TLSDesc, TLSInfo); 520 return Error::success(); 521 } 522 523 } // namespace 524 525 namespace llvm { 526 namespace jitlink { 527 528 Expected<std::unique_ptr<LinkGraph>> 529 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { 530 LLVM_DEBUG({ 531 dbgs() << "Building jitlink graph for new input " 532 << ObjectBuffer.getBufferIdentifier() << "...\n"; 533 }); 534 535 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 536 if (!ELFObj) 537 return ELFObj.takeError(); 538 539 assert((*ELFObj)->getArch() == Triple::aarch64 && 540 "Only AArch64 (little endian) is supported for now"); 541 542 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 543 return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(), 544 ELFObjFile.getELFFile(), 545 (*ELFObj)->makeTriple()) 546 .buildGraph(); 547 } 548 549 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G, 550 std::unique_ptr<JITLinkContext> Ctx) { 551 PassConfiguration Config; 552 const Triple &TT = G->getTargetTriple(); 553 if (Ctx->shouldAddDefaultTargetPasses(TT)) { 554 // Add eh-frame passses. 555 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); 556 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 557 ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, 558 aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); 559 560 // Add a mark-live pass. 561 if (auto MarkLive = Ctx->getMarkLivePass(TT)) 562 Config.PrePrunePasses.push_back(std::move(MarkLive)); 563 else 564 Config.PrePrunePasses.push_back(markAllSymbolsLive); 565 566 // Add an in-place GOT/TLS/Stubs build pass. 567 Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); 568 } 569 570 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 571 return Ctx->notifyFailed(std::move(Err)); 572 573 ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config)); 574 } 575 576 } // namespace jitlink 577 } // namespace llvm 578