1 //===----- ELF_aarch64.cpp - JIT linker implementation for ELF/aarch64 ----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // ELF/aarch64 jit-link implementation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" 14 #include "EHFrameSupportImpl.h" 15 #include "ELFLinkGraphBuilder.h" 16 #include "JITLinkGeneric.h" 17 #include "llvm/BinaryFormat/ELF.h" 18 #include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" 19 #include "llvm/ExecutionEngine/JITLink/aarch64.h" 20 #include "llvm/Object/ELFObjectFile.h" 21 #include "llvm/Support/Endian.h" 22 23 #define DEBUG_TYPE "jitlink" 24 25 using namespace llvm; 26 using namespace llvm::jitlink; 27 28 namespace { 29 30 class ELFJITLinker_aarch64 : public JITLinker<ELFJITLinker_aarch64> { 31 friend class JITLinker<ELFJITLinker_aarch64>; 32 33 public: 34 ELFJITLinker_aarch64(std::unique_ptr<JITLinkContext> Ctx, 35 std::unique_ptr<LinkGraph> G, 36 PassConfiguration PassConfig) 37 : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} 38 39 private: 40 Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { 41 return aarch64::applyFixup(G, B, E); 42 } 43 }; 44 45 template <typename ELFT> 46 class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> { 47 private: 48 enum ELFAArch64RelocationKind : Edge::Kind { 49 ELFCall26 = Edge::FirstRelocation, 50 ELFAdrPage21, 51 ELFAddAbs12, 52 ELFLdSt8Abs12, 53 ELFLdSt16Abs12, 54 ELFLdSt32Abs12, 55 ELFLdSt64Abs12, 56 ELFLdSt128Abs12, 57 ELFMovwAbsG0, 58 ELFMovwAbsG1, 59 ELFMovwAbsG2, 60 ELFMovwAbsG3, 61 ELFAbs64, 62 ELFPrel32, 63 ELFPrel64, 64 ELFAdrGOTPage21, 65 ELFLd64GOTLo12, 66 ELFTLSDescAdrPage21, 67 ELFTLSDescAddLo12, 68 ELFTLSDescLd64Lo12, 69 ELFTLSDescCall, 70 }; 71 72 static Expected<ELFAArch64RelocationKind> 73 getRelocationKind(const uint32_t Type) { 74 using namespace aarch64; 75 switch (Type) { 76 case ELF::R_AARCH64_CALL26: 77 case ELF::R_AARCH64_JUMP26: 78 return ELFCall26; 79 case ELF::R_AARCH64_ADR_PREL_PG_HI21: 80 return ELFAdrPage21; 81 case ELF::R_AARCH64_ADD_ABS_LO12_NC: 82 return ELFAddAbs12; 83 case ELF::R_AARCH64_LDST8_ABS_LO12_NC: 84 return ELFLdSt8Abs12; 85 case ELF::R_AARCH64_LDST16_ABS_LO12_NC: 86 return ELFLdSt16Abs12; 87 case ELF::R_AARCH64_LDST32_ABS_LO12_NC: 88 return ELFLdSt32Abs12; 89 case ELF::R_AARCH64_LDST64_ABS_LO12_NC: 90 return ELFLdSt64Abs12; 91 case ELF::R_AARCH64_LDST128_ABS_LO12_NC: 92 return ELFLdSt128Abs12; 93 case ELF::R_AARCH64_MOVW_UABS_G0_NC: 94 return ELFMovwAbsG0; 95 case ELF::R_AARCH64_MOVW_UABS_G1_NC: 96 return ELFMovwAbsG1; 97 case ELF::R_AARCH64_MOVW_UABS_G2_NC: 98 return ELFMovwAbsG2; 99 case ELF::R_AARCH64_MOVW_UABS_G3: 100 return ELFMovwAbsG3; 101 case ELF::R_AARCH64_ABS64: 102 return ELFAbs64; 103 case ELF::R_AARCH64_PREL32: 104 return ELFPrel32; 105 case ELF::R_AARCH64_PREL64: 106 return ELFPrel64; 107 case ELF::R_AARCH64_ADR_GOT_PAGE: 108 return ELFAdrGOTPage21; 109 case ELF::R_AARCH64_LD64_GOT_LO12_NC: 110 return ELFLd64GOTLo12; 111 case ELF::R_AARCH64_TLSDESC_ADR_PAGE21: 112 return ELFTLSDescAdrPage21; 113 case ELF::R_AARCH64_TLSDESC_ADD_LO12: 114 return ELFTLSDescAddLo12; 115 case ELF::R_AARCH64_TLSDESC_LD64_LO12: 116 return ELFTLSDescLd64Lo12; 117 case ELF::R_AARCH64_TLSDESC_CALL: 118 return ELFTLSDescCall; 119 } 120 121 return make_error<JITLinkError>( 122 "Unsupported aarch64 relocation:" + formatv("{0:d}: ", Type) + 123 object::getELFRelocationTypeName(ELF::EM_AARCH64, Type)); 124 } 125 126 Error addRelocations() override { 127 LLVM_DEBUG(dbgs() << "Processing relocations:\n"); 128 129 using Base = ELFLinkGraphBuilder<ELFT>; 130 using Self = ELFLinkGraphBuilder_aarch64<ELFT>; 131 for (const auto &RelSect : Base::Sections) 132 if (Error Err = Base::forEachRelaRelocation(RelSect, this, 133 &Self::addSingleRelocation)) 134 return Err; 135 136 return Error::success(); 137 } 138 139 Error addSingleRelocation(const typename ELFT::Rela &Rel, 140 const typename ELFT::Shdr &FixupSect, 141 Block &BlockToFix) { 142 using support::ulittle32_t; 143 using Base = ELFLinkGraphBuilder<ELFT>; 144 145 uint32_t SymbolIndex = Rel.getSymbol(false); 146 auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); 147 if (!ObjSymbol) 148 return ObjSymbol.takeError(); 149 150 Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); 151 if (!GraphSymbol) 152 return make_error<StringError>( 153 formatv("Could not find symbol at given index, did you add it to " 154 "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", 155 SymbolIndex, (*ObjSymbol)->st_shndx, 156 Base::GraphSymbols.size()), 157 inconvertibleErrorCode()); 158 159 uint32_t Type = Rel.getType(false); 160 Expected<ELFAArch64RelocationKind> RelocKind = getRelocationKind(Type); 161 if (!RelocKind) 162 return RelocKind.takeError(); 163 164 int64_t Addend = Rel.r_addend; 165 orc::ExecutorAddr FixupAddress = 166 orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; 167 Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); 168 169 // Get a pointer to the fixup content. 170 const void *FixupContent = BlockToFix.getContent().data() + 171 (FixupAddress - BlockToFix.getAddress()); 172 173 Edge::Kind Kind = Edge::Invalid; 174 175 switch (*RelocKind) { 176 case ELFCall26: { 177 Kind = aarch64::Branch26PCRel; 178 break; 179 } 180 case ELFAdrPage21: { 181 Kind = aarch64::Page21; 182 break; 183 } 184 case ELFAddAbs12: { 185 Kind = aarch64::PageOffset12; 186 break; 187 } 188 case ELFLdSt8Abs12: { 189 uint32_t Instr = *(const ulittle32_t *)FixupContent; 190 if (!aarch64::isLoadStoreImm12(Instr) || 191 aarch64::getPageOffset12Shift(Instr) != 0) 192 return make_error<JITLinkError>( 193 "R_AARCH64_LDST8_ABS_LO12_NC target is not a " 194 "LDRB/STRB (imm12) instruction"); 195 196 Kind = aarch64::PageOffset12; 197 break; 198 } 199 case ELFLdSt16Abs12: { 200 uint32_t Instr = *(const ulittle32_t *)FixupContent; 201 if (!aarch64::isLoadStoreImm12(Instr) || 202 aarch64::getPageOffset12Shift(Instr) != 1) 203 return make_error<JITLinkError>( 204 "R_AARCH64_LDST16_ABS_LO12_NC target is not a " 205 "LDRH/STRH (imm12) instruction"); 206 207 Kind = aarch64::PageOffset12; 208 break; 209 } 210 case ELFLdSt32Abs12: { 211 uint32_t Instr = *(const ulittle32_t *)FixupContent; 212 if (!aarch64::isLoadStoreImm12(Instr) || 213 aarch64::getPageOffset12Shift(Instr) != 2) 214 return make_error<JITLinkError>( 215 "R_AARCH64_LDST32_ABS_LO12_NC target is not a " 216 "LDR/STR (imm12, 32 bit) instruction"); 217 218 Kind = aarch64::PageOffset12; 219 break; 220 } 221 case ELFLdSt64Abs12: { 222 uint32_t Instr = *(const ulittle32_t *)FixupContent; 223 if (!aarch64::isLoadStoreImm12(Instr) || 224 aarch64::getPageOffset12Shift(Instr) != 3) 225 return make_error<JITLinkError>( 226 "R_AARCH64_LDST64_ABS_LO12_NC target is not a " 227 "LDR/STR (imm12, 64 bit) instruction"); 228 229 Kind = aarch64::PageOffset12; 230 break; 231 } 232 case ELFLdSt128Abs12: { 233 uint32_t Instr = *(const ulittle32_t *)FixupContent; 234 if (!aarch64::isLoadStoreImm12(Instr) || 235 aarch64::getPageOffset12Shift(Instr) != 4) 236 return make_error<JITLinkError>( 237 "R_AARCH64_LDST128_ABS_LO12_NC target is not a " 238 "LDR/STR (imm12, 128 bit) instruction"); 239 240 Kind = aarch64::PageOffset12; 241 break; 242 } 243 case ELFMovwAbsG0: { 244 uint32_t Instr = *(const ulittle32_t *)FixupContent; 245 if (!aarch64::isMoveWideImm16(Instr) || 246 aarch64::getMoveWide16Shift(Instr) != 0) 247 return make_error<JITLinkError>( 248 "R_AARCH64_MOVW_UABS_G0_NC target is not a " 249 "MOVK/MOVZ (imm16, LSL #0) instruction"); 250 251 Kind = aarch64::MoveWide16; 252 break; 253 } 254 case ELFMovwAbsG1: { 255 uint32_t Instr = *(const ulittle32_t *)FixupContent; 256 if (!aarch64::isMoveWideImm16(Instr) || 257 aarch64::getMoveWide16Shift(Instr) != 16) 258 return make_error<JITLinkError>( 259 "R_AARCH64_MOVW_UABS_G1_NC target is not a " 260 "MOVK/MOVZ (imm16, LSL #16) instruction"); 261 262 Kind = aarch64::MoveWide16; 263 break; 264 } 265 case ELFMovwAbsG2: { 266 uint32_t Instr = *(const ulittle32_t *)FixupContent; 267 if (!aarch64::isMoveWideImm16(Instr) || 268 aarch64::getMoveWide16Shift(Instr) != 32) 269 return make_error<JITLinkError>( 270 "R_AARCH64_MOVW_UABS_G2_NC target is not a " 271 "MOVK/MOVZ (imm16, LSL #32) instruction"); 272 273 Kind = aarch64::MoveWide16; 274 break; 275 } 276 case ELFMovwAbsG3: { 277 uint32_t Instr = *(const ulittle32_t *)FixupContent; 278 if (!aarch64::isMoveWideImm16(Instr) || 279 aarch64::getMoveWide16Shift(Instr) != 48) 280 return make_error<JITLinkError>( 281 "R_AARCH64_MOVW_UABS_G3 target is not a " 282 "MOVK/MOVZ (imm16, LSL #48) instruction"); 283 284 Kind = aarch64::MoveWide16; 285 break; 286 } 287 case ELFAbs64: { 288 Kind = aarch64::Pointer64; 289 break; 290 } 291 case ELFPrel32: { 292 Kind = aarch64::Delta32; 293 break; 294 } 295 case ELFPrel64: { 296 Kind = aarch64::Delta64; 297 break; 298 } 299 case ELFAdrGOTPage21: { 300 Kind = aarch64::RequestGOTAndTransformToPage21; 301 break; 302 } 303 case ELFLd64GOTLo12: { 304 Kind = aarch64::RequestGOTAndTransformToPageOffset12; 305 break; 306 } 307 case ELFTLSDescAdrPage21: { 308 Kind = aarch64::RequestTLSDescEntryAndTransformToPage21; 309 break; 310 } 311 case ELFTLSDescAddLo12: 312 case ELFTLSDescLd64Lo12: { 313 Kind = aarch64::RequestTLSDescEntryAndTransformToPageOffset12; 314 break; 315 } 316 case ELFTLSDescCall: { 317 return Error::success(); 318 } 319 }; 320 321 Edge GE(Kind, Offset, *GraphSymbol, Addend); 322 LLVM_DEBUG({ 323 dbgs() << " "; 324 printEdge(dbgs(), BlockToFix, GE, aarch64::getEdgeKindName(Kind)); 325 dbgs() << "\n"; 326 }); 327 328 BlockToFix.addEdge(std::move(GE)); 329 330 return Error::success(); 331 } 332 333 /// Return the string name of the given ELF aarch64 edge kind. 334 const char *getELFAArch64RelocationKindName(Edge::Kind R) { 335 switch (R) { 336 case ELFCall26: 337 return "ELFCall26"; 338 case ELFAdrPage21: 339 return "ELFAdrPage21"; 340 case ELFAddAbs12: 341 return "ELFAddAbs12"; 342 case ELFLdSt8Abs12: 343 return "ELFLdSt8Abs12"; 344 case ELFLdSt16Abs12: 345 return "ELFLdSt16Abs12"; 346 case ELFLdSt32Abs12: 347 return "ELFLdSt32Abs12"; 348 case ELFLdSt64Abs12: 349 return "ELFLdSt64Abs12"; 350 case ELFLdSt128Abs12: 351 return "ELFLdSt128Abs12"; 352 case ELFMovwAbsG0: 353 return "ELFMovwAbsG0"; 354 case ELFMovwAbsG1: 355 return "ELFMovwAbsG1"; 356 case ELFMovwAbsG2: 357 return "ELFMovwAbsG2"; 358 case ELFMovwAbsG3: 359 return "ELFMovwAbsG3"; 360 case ELFAbs64: 361 return "ELFAbs64"; 362 case ELFPrel32: 363 return "ELFPrel32"; 364 case ELFPrel64: 365 return "ELFPrel64"; 366 case ELFAdrGOTPage21: 367 return "ELFAdrGOTPage21"; 368 case ELFLd64GOTLo12: 369 return "ELFLd64GOTLo12"; 370 case ELFTLSDescAdrPage21: 371 return "ELFTLSDescAdrPage21"; 372 case ELFTLSDescAddLo12: 373 return "ELFTLSDescAddLo12"; 374 case ELFTLSDescLd64Lo12: 375 return "ELFTLSDescLd64Lo12"; 376 case ELFTLSDescCall: 377 return "ELFTLSDescCall"; 378 default: 379 return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); 380 } 381 } 382 383 public: 384 ELFLinkGraphBuilder_aarch64(StringRef FileName, 385 const object::ELFFile<ELFT> &Obj, const Triple T) 386 : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, 387 aarch64::getEdgeKindName) {} 388 }; 389 390 // TLS Info Builder. 391 class TLSInfoTableManager_ELF_aarch64 392 : public TableManager<TLSInfoTableManager_ELF_aarch64> { 393 public: 394 static StringRef getSectionName() { return "$__TLSINFO"; } 395 396 static const uint8_t TLSInfoEntryContent[16]; 397 398 bool visitEdge(LinkGraph &G, Block *B, Edge &E) { return false; } 399 400 Symbol &createEntry(LinkGraph &G, Symbol &Target) { 401 // the TLS Info entry's key value will be written by the fixTLVSectionByName 402 // pass, so create mutable content. 403 auto &TLSInfoEntry = G.createMutableContentBlock( 404 getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), 405 orc::ExecutorAddr(), 8, 0); 406 TLSInfoEntry.addEdge(aarch64::Pointer64, 8, Target, 0); 407 return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); 408 } 409 410 private: 411 Section &getTLSInfoSection(LinkGraph &G) { 412 if (!TLSInfoTable) 413 TLSInfoTable = &G.createSection(getSectionName(), orc::MemProt::Read); 414 return *TLSInfoTable; 415 } 416 417 ArrayRef<char> getTLSInfoEntryContent() const { 418 return {reinterpret_cast<const char *>(TLSInfoEntryContent), 419 sizeof(TLSInfoEntryContent)}; 420 } 421 422 Section *TLSInfoTable = nullptr; 423 }; 424 425 const uint8_t TLSInfoTableManager_ELF_aarch64::TLSInfoEntryContent[16] = { 426 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ 427 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ 428 }; 429 430 // TLS Descriptor Builder. 431 class TLSDescTableManager_ELF_aarch64 432 : public TableManager<TLSDescTableManager_ELF_aarch64> { 433 public: 434 TLSDescTableManager_ELF_aarch64( 435 TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager) 436 : TLSInfoTableManager(TLSInfoTableManager) {} 437 438 static StringRef getSectionName() { return "$__TLSDESC"; } 439 440 static const uint8_t TLSDescEntryContent[16]; 441 442 bool visitEdge(LinkGraph &G, Block *B, Edge &E) { 443 Edge::Kind KindToSet = Edge::Invalid; 444 switch (E.getKind()) { 445 case aarch64::RequestTLSDescEntryAndTransformToPage21: { 446 KindToSet = aarch64::Page21; 447 break; 448 } 449 case aarch64::RequestTLSDescEntryAndTransformToPageOffset12: { 450 KindToSet = aarch64::PageOffset12; 451 break; 452 } 453 default: 454 return false; 455 } 456 assert(KindToSet != Edge::Invalid && 457 "Fell through switch, but no new kind to set"); 458 DEBUG_WITH_TYPE("jitlink", { 459 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " 460 << B->getFixupAddress(E) << " (" << B->getAddress() << " + " 461 << formatv("{0:x}", E.getOffset()) << ")\n"; 462 }); 463 E.setKind(KindToSet); 464 E.setTarget(getEntryForTarget(G, E.getTarget())); 465 return true; 466 } 467 468 Symbol &createEntry(LinkGraph &G, Symbol &Target) { 469 auto &EntryBlock = 470 G.createContentBlock(getTLSDescSection(G), getTLSDescBlockContent(), 471 orc::ExecutorAddr(), 8, 0); 472 EntryBlock.addEdge(aarch64::Pointer64, 0, getTLSDescResolver(G), 0); 473 EntryBlock.addEdge(aarch64::Pointer64, 8, 474 TLSInfoTableManager.getEntryForTarget(G, Target), 0); 475 return G.addAnonymousSymbol(EntryBlock, 0, 8, false, false); 476 } 477 478 private: 479 Section &getTLSDescSection(LinkGraph &G) { 480 if (!GOTSection) 481 GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read); 482 return *GOTSection; 483 } 484 485 Symbol &getTLSDescResolver(LinkGraph &G) { 486 if (!TLSDescResolver) 487 TLSDescResolver = &G.addExternalSymbol("__tlsdesc_resolver", 8, false); 488 return *TLSDescResolver; 489 } 490 491 ArrayRef<char> getTLSDescBlockContent() { 492 return {reinterpret_cast<const char *>(TLSDescEntryContent), 493 sizeof(TLSDescEntryContent)}; 494 } 495 496 Section *GOTSection = nullptr; 497 Symbol *TLSDescResolver = nullptr; 498 TLSInfoTableManager_ELF_aarch64 &TLSInfoTableManager; 499 }; 500 501 const uint8_t TLSDescTableManager_ELF_aarch64::TLSDescEntryContent[16] = { 502 0x00, 0x00, 0x00, 0x00, 503 0x00, 0x00, 0x00, 0x00, /*resolver function pointer*/ 504 0x00, 0x00, 0x00, 0x00, 505 0x00, 0x00, 0x00, 0x00 /*pointer to tls info*/ 506 }; 507 508 Error buildTables_ELF_aarch64(LinkGraph &G) { 509 LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); 510 511 aarch64::GOTTableManager GOT; 512 aarch64::PLTTableManager PLT(GOT); 513 TLSInfoTableManager_ELF_aarch64 TLSInfo; 514 TLSDescTableManager_ELF_aarch64 TLSDesc(TLSInfo); 515 visitExistingEdges(G, GOT, PLT, TLSDesc, TLSInfo); 516 return Error::success(); 517 } 518 519 } // namespace 520 521 namespace llvm { 522 namespace jitlink { 523 524 Expected<std::unique_ptr<LinkGraph>> 525 createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { 526 LLVM_DEBUG({ 527 dbgs() << "Building jitlink graph for new input " 528 << ObjectBuffer.getBufferIdentifier() << "...\n"; 529 }); 530 531 auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); 532 if (!ELFObj) 533 return ELFObj.takeError(); 534 535 assert((*ELFObj)->getArch() == Triple::aarch64 && 536 "Only AArch64 (little endian) is supported for now"); 537 538 auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); 539 return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(), 540 ELFObjFile.getELFFile(), 541 (*ELFObj)->makeTriple()) 542 .buildGraph(); 543 } 544 545 void link_ELF_aarch64(std::unique_ptr<LinkGraph> G, 546 std::unique_ptr<JITLinkContext> Ctx) { 547 PassConfiguration Config; 548 const Triple &TT = G->getTargetTriple(); 549 if (Ctx->shouldAddDefaultTargetPasses(TT)) { 550 // Add eh-frame passses. 551 Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); 552 Config.PrePrunePasses.push_back(EHFrameEdgeFixer( 553 ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, 554 aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); 555 Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); 556 557 // Add a mark-live pass. 558 if (auto MarkLive = Ctx->getMarkLivePass(TT)) 559 Config.PrePrunePasses.push_back(std::move(MarkLive)); 560 else 561 Config.PrePrunePasses.push_back(markAllSymbolsLive); 562 563 // Add an in-place GOT/TLS/Stubs build pass. 564 Config.PostPrunePasses.push_back(buildTables_ELF_aarch64); 565 } 566 567 if (auto Err = Ctx->modifyPassConfig(*G, Config)) 568 return Ctx->notifyFailed(std::move(Err)); 569 570 ELFJITLinker_aarch64::link(std::move(Ctx), std::move(G), std::move(Config)); 571 } 572 573 } // namespace jitlink 574 } // namespace llvm 575