1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "OutputSections.h" 10 #include "Symbols.h" 11 #include "SyntheticSections.h" 12 #include "Target.h" 13 #include "lld/Common/ErrorHandler.h" 14 #include "llvm/BinaryFormat/ELF.h" 15 #include "llvm/Support/Endian.h" 16 17 using namespace llvm; 18 using namespace llvm::support::endian; 19 using namespace llvm::ELF; 20 using namespace lld; 21 using namespace lld::elf; 22 23 // Page(Expr) is the page address of the expression Expr, defined 24 // as (Expr & ~0xFFF). (This applies even if the machine page size 25 // supported by the platform has a different value.) 26 uint64_t elf::getAArch64Page(uint64_t expr) { 27 return expr & ~static_cast<uint64_t>(0xFFF); 28 } 29 30 namespace { 31 class AArch64 : public TargetInfo { 32 public: 33 AArch64(); 34 RelExpr getRelExpr(RelType type, const Symbol &s, 35 const uint8_t *loc) const override; 36 RelType getDynRel(RelType type) const override; 37 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; 38 void writeGotPlt(uint8_t *buf, const Symbol &s) const override; 39 void writeIgotPlt(uint8_t *buf, const Symbol &s) const override; 40 void writePltHeader(uint8_t *buf) const override; 41 void writePlt(uint8_t *buf, const Symbol &sym, 42 uint64_t pltEntryAddr) const override; 43 bool needsThunk(RelExpr expr, RelType type, const InputFile *file, 44 uint64_t branchAddr, const Symbol &s, 45 int64_t a) const override; 46 uint32_t getThunkSectionSpacing() const override; 47 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; 48 bool usesOnlyLowPageBits(RelType type) const override; 49 void relocate(uint8_t *loc, const Relocation &rel, 50 uint64_t val) const override; 51 RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; 52 void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; 53 54 private: 55 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; 56 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; 57 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; 58 }; 59 60 struct AArch64Relaxer { 61 bool safeToRelaxAdrpLdr = false; 62 63 AArch64Relaxer(ArrayRef<Relocation> relocs); 64 bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel, 65 uint64_t secAddr, uint8_t *buf) const; 66 bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel, 67 uint64_t secAddr, uint8_t *buf) const; 68 }; 69 } // namespace 70 71 AArch64::AArch64() { 72 copyRel = R_AARCH64_COPY; 73 relativeRel = R_AARCH64_RELATIVE; 74 iRelativeRel = R_AARCH64_IRELATIVE; 75 gotRel = R_AARCH64_GLOB_DAT; 76 pltRel = R_AARCH64_JUMP_SLOT; 77 symbolicRel = R_AARCH64_ABS64; 78 tlsDescRel = R_AARCH64_TLSDESC; 79 tlsGotRel = R_AARCH64_TLS_TPREL64; 80 pltHeaderSize = 32; 81 pltEntrySize = 16; 82 ipltEntrySize = 16; 83 defaultMaxPageSize = 65536; 84 85 // Align to the 2 MiB page size (known as a superpage or huge page). 86 // FreeBSD automatically promotes 2 MiB-aligned allocations. 87 defaultImageBase = 0x200000; 88 89 needsThunks = true; 90 } 91 92 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, 93 const uint8_t *loc) const { 94 switch (type) { 95 case R_AARCH64_ABS16: 96 case R_AARCH64_ABS32: 97 case R_AARCH64_ABS64: 98 case R_AARCH64_ADD_ABS_LO12_NC: 99 case R_AARCH64_LDST128_ABS_LO12_NC: 100 case R_AARCH64_LDST16_ABS_LO12_NC: 101 case R_AARCH64_LDST32_ABS_LO12_NC: 102 case R_AARCH64_LDST64_ABS_LO12_NC: 103 case R_AARCH64_LDST8_ABS_LO12_NC: 104 case R_AARCH64_MOVW_SABS_G0: 105 case R_AARCH64_MOVW_SABS_G1: 106 case R_AARCH64_MOVW_SABS_G2: 107 case R_AARCH64_MOVW_UABS_G0: 108 case R_AARCH64_MOVW_UABS_G0_NC: 109 case R_AARCH64_MOVW_UABS_G1: 110 case R_AARCH64_MOVW_UABS_G1_NC: 111 case R_AARCH64_MOVW_UABS_G2: 112 case R_AARCH64_MOVW_UABS_G2_NC: 113 case R_AARCH64_MOVW_UABS_G3: 114 return R_ABS; 115 case R_AARCH64_TLSDESC_ADR_PAGE21: 116 return R_AARCH64_TLSDESC_PAGE; 117 case R_AARCH64_TLSDESC_LD64_LO12: 118 case R_AARCH64_TLSDESC_ADD_LO12: 119 return R_TLSDESC; 120 case R_AARCH64_TLSDESC_CALL: 121 return R_TLSDESC_CALL; 122 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 123 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 124 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 125 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 126 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 127 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 128 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 129 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 130 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 131 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 132 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 133 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 134 return R_TPREL; 135 case R_AARCH64_CALL26: 136 case R_AARCH64_CONDBR19: 137 case R_AARCH64_JUMP26: 138 case R_AARCH64_TSTBR14: 139 return R_PLT_PC; 140 case R_AARCH64_PLT32: 141 const_cast<Symbol &>(s).thunkAccessed = true; 142 return R_PLT_PC; 143 case R_AARCH64_PREL16: 144 case R_AARCH64_PREL32: 145 case R_AARCH64_PREL64: 146 case R_AARCH64_ADR_PREL_LO21: 147 case R_AARCH64_LD_PREL_LO19: 148 case R_AARCH64_MOVW_PREL_G0: 149 case R_AARCH64_MOVW_PREL_G0_NC: 150 case R_AARCH64_MOVW_PREL_G1: 151 case R_AARCH64_MOVW_PREL_G1_NC: 152 case R_AARCH64_MOVW_PREL_G2: 153 case R_AARCH64_MOVW_PREL_G2_NC: 154 case R_AARCH64_MOVW_PREL_G3: 155 return R_PC; 156 case R_AARCH64_ADR_PREL_PG_HI21: 157 case R_AARCH64_ADR_PREL_PG_HI21_NC: 158 return R_AARCH64_PAGE_PC; 159 case R_AARCH64_LD64_GOT_LO12_NC: 160 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 161 return R_GOT; 162 case R_AARCH64_LD64_GOTPAGE_LO15: 163 return R_AARCH64_GOT_PAGE; 164 case R_AARCH64_ADR_GOT_PAGE: 165 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 166 return R_AARCH64_GOT_PAGE_PC; 167 case R_AARCH64_NONE: 168 return R_NONE; 169 default: 170 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + 171 ") against symbol " + toString(s)); 172 return R_NONE; 173 } 174 } 175 176 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const { 177 if (expr == R_RELAX_TLS_GD_TO_IE) { 178 if (type == R_AARCH64_TLSDESC_ADR_PAGE21) 179 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; 180 return R_RELAX_TLS_GD_TO_IE_ABS; 181 } 182 return expr; 183 } 184 185 bool AArch64::usesOnlyLowPageBits(RelType type) const { 186 switch (type) { 187 default: 188 return false; 189 case R_AARCH64_ADD_ABS_LO12_NC: 190 case R_AARCH64_LD64_GOT_LO12_NC: 191 case R_AARCH64_LDST128_ABS_LO12_NC: 192 case R_AARCH64_LDST16_ABS_LO12_NC: 193 case R_AARCH64_LDST32_ABS_LO12_NC: 194 case R_AARCH64_LDST64_ABS_LO12_NC: 195 case R_AARCH64_LDST8_ABS_LO12_NC: 196 case R_AARCH64_TLSDESC_ADD_LO12: 197 case R_AARCH64_TLSDESC_LD64_LO12: 198 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 199 return true; 200 } 201 } 202 203 RelType AArch64::getDynRel(RelType type) const { 204 if (type == R_AARCH64_ABS64) 205 return type; 206 return R_AARCH64_NONE; 207 } 208 209 int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const { 210 switch (type) { 211 case R_AARCH64_TLSDESC: 212 return read64(buf + 8); 213 case R_AARCH64_NONE: 214 case R_AARCH64_GLOB_DAT: 215 case R_AARCH64_JUMP_SLOT: 216 return 0; 217 case R_AARCH64_PREL32: 218 return SignExtend64<32>(read32(buf)); 219 case R_AARCH64_ABS64: 220 case R_AARCH64_PREL64: 221 case R_AARCH64_RELATIVE: 222 case R_AARCH64_IRELATIVE: 223 case R_AARCH64_TLS_TPREL64: 224 return read64(buf); 225 default: 226 internalLinkerError(getErrorLocation(buf), 227 "cannot read addend for relocation " + toString(type)); 228 return 0; 229 } 230 } 231 232 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const { 233 write64(buf, in.plt->getVA()); 234 } 235 236 void AArch64::writeIgotPlt(uint8_t *buf, const Symbol &s) const { 237 if (config->writeAddends) 238 write64(buf, s.getVA()); 239 } 240 241 void AArch64::writePltHeader(uint8_t *buf) const { 242 const uint8_t pltData[] = { 243 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 244 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2])) 245 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))] 246 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2])) 247 0x20, 0x02, 0x1f, 0xd6, // br x17 248 0x1f, 0x20, 0x03, 0xd5, // nop 249 0x1f, 0x20, 0x03, 0xd5, // nop 250 0x1f, 0x20, 0x03, 0xd5 // nop 251 }; 252 memcpy(buf, pltData, sizeof(pltData)); 253 254 uint64_t got = in.gotPlt->getVA(); 255 uint64_t plt = in.plt->getVA(); 256 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 257 getAArch64Page(got + 16) - getAArch64Page(plt + 4)); 258 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 259 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 260 } 261 262 void AArch64::writePlt(uint8_t *buf, const Symbol &sym, 263 uint64_t pltEntryAddr) const { 264 const uint8_t inst[] = { 265 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n])) 266 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))] 267 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[n])) 268 0x20, 0x02, 0x1f, 0xd6 // br x17 269 }; 270 memcpy(buf, inst, sizeof(inst)); 271 272 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 273 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21, 274 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 275 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 276 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 277 } 278 279 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file, 280 uint64_t branchAddr, const Symbol &s, 281 int64_t a) const { 282 // If s is an undefined weak symbol and does not have a PLT entry then it will 283 // be resolved as a branch to the next instruction. If it is hidden, its 284 // binding has been converted to local, so we just check isUndefined() here. A 285 // undefined non-weak symbol will have been errored. 286 if (s.isUndefined() && !s.isInPlt()) 287 return false; 288 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 289 // only permits range extension thunks for R_AARCH64_CALL26 and 290 // R_AARCH64_JUMP26 relocation types. 291 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && 292 type != R_AARCH64_PLT32) 293 return false; 294 uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a); 295 return !inBranchRange(type, branchAddr, dst); 296 } 297 298 uint32_t AArch64::getThunkSectionSpacing() const { 299 // See comment in Arch/ARM.cpp for a more detailed explanation of 300 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 301 // Thunk have a range of +/- 128 MiB 302 return (128 * 1024 * 1024) - 0x30000; 303 } 304 305 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { 306 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && 307 type != R_AARCH64_PLT32) 308 return true; 309 // The AArch64 call and unconditional branch instructions have a range of 310 // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB. 311 uint64_t range = 312 type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024); 313 if (dst > src) { 314 // Immediate of branch is signed. 315 range -= 4; 316 return dst - src <= range; 317 } 318 return src - dst <= range; 319 } 320 321 static void write32AArch64Addr(uint8_t *l, uint64_t imm) { 322 uint32_t immLo = (imm & 0x3) << 29; 323 uint32_t immHi = (imm & 0x1FFFFC) << 3; 324 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3); 325 write32le(l, (read32le(l) & ~mask) | immLo | immHi); 326 } 327 328 // Return the bits [Start, End] from Val shifted Start bits. 329 // For instance, getBits(0xF0, 4, 8) returns 0xF. 330 static uint64_t getBits(uint64_t val, int start, int end) { 331 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1; 332 return (val >> start) & mask; 333 } 334 335 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); } 336 337 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 338 static void or32AArch64Imm(uint8_t *l, uint64_t imm) { 339 or32le(l, (imm & 0xFFF) << 10); 340 } 341 342 // Update the immediate field in an AArch64 movk, movn or movz instruction 343 // for a signed relocation, and update the opcode of a movn or movz instruction 344 // to match the sign of the operand. 345 static void writeSMovWImm(uint8_t *loc, uint32_t imm) { 346 uint32_t inst = read32le(loc); 347 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. 348 if (!(inst & (1 << 29))) { 349 // movn or movz. 350 if (imm & 0x10000) { 351 // Change opcode to movn, which takes an inverted operand. 352 imm ^= 0xFFFF; 353 inst &= ~(1 << 30); 354 } else { 355 // Change opcode to movz. 356 inst |= 1 << 30; 357 } 358 } 359 write32le(loc, inst | ((imm & 0xFFFF) << 5)); 360 } 361 362 void AArch64::relocate(uint8_t *loc, const Relocation &rel, 363 uint64_t val) const { 364 switch (rel.type) { 365 case R_AARCH64_ABS16: 366 case R_AARCH64_PREL16: 367 checkIntUInt(loc, val, 16, rel); 368 write16(loc, val); 369 break; 370 case R_AARCH64_ABS32: 371 case R_AARCH64_PREL32: 372 checkIntUInt(loc, val, 32, rel); 373 write32(loc, val); 374 break; 375 case R_AARCH64_PLT32: 376 checkInt(loc, val, 32, rel); 377 write32(loc, val); 378 break; 379 case R_AARCH64_ABS64: 380 case R_AARCH64_PREL64: 381 write64(loc, val); 382 break; 383 case R_AARCH64_ADD_ABS_LO12_NC: 384 or32AArch64Imm(loc, val); 385 break; 386 case R_AARCH64_ADR_GOT_PAGE: 387 case R_AARCH64_ADR_PREL_PG_HI21: 388 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 389 case R_AARCH64_TLSDESC_ADR_PAGE21: 390 checkInt(loc, val, 33, rel); 391 [[fallthrough]]; 392 case R_AARCH64_ADR_PREL_PG_HI21_NC: 393 write32AArch64Addr(loc, val >> 12); 394 break; 395 case R_AARCH64_ADR_PREL_LO21: 396 checkInt(loc, val, 21, rel); 397 write32AArch64Addr(loc, val); 398 break; 399 case R_AARCH64_JUMP26: 400 // Normally we would just write the bits of the immediate field, however 401 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 402 // we want to replace a non-branch instruction with a branch immediate 403 // instruction. By writing all the bits of the instruction including the 404 // opcode and the immediate (0 001 | 01 imm26) we can do this 405 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 406 // the instruction we want to patch. 407 write32le(loc, 0x14000000); 408 [[fallthrough]]; 409 case R_AARCH64_CALL26: 410 checkInt(loc, val, 28, rel); 411 or32le(loc, (val & 0x0FFFFFFC) >> 2); 412 break; 413 case R_AARCH64_CONDBR19: 414 case R_AARCH64_LD_PREL_LO19: 415 checkAlignment(loc, val, 4, rel); 416 checkInt(loc, val, 21, rel); 417 or32le(loc, (val & 0x1FFFFC) << 3); 418 break; 419 case R_AARCH64_LDST8_ABS_LO12_NC: 420 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 421 or32AArch64Imm(loc, getBits(val, 0, 11)); 422 break; 423 case R_AARCH64_LDST16_ABS_LO12_NC: 424 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 425 checkAlignment(loc, val, 2, rel); 426 or32AArch64Imm(loc, getBits(val, 1, 11)); 427 break; 428 case R_AARCH64_LDST32_ABS_LO12_NC: 429 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 430 checkAlignment(loc, val, 4, rel); 431 or32AArch64Imm(loc, getBits(val, 2, 11)); 432 break; 433 case R_AARCH64_LDST64_ABS_LO12_NC: 434 case R_AARCH64_LD64_GOT_LO12_NC: 435 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 436 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 437 case R_AARCH64_TLSDESC_LD64_LO12: 438 checkAlignment(loc, val, 8, rel); 439 or32AArch64Imm(loc, getBits(val, 3, 11)); 440 break; 441 case R_AARCH64_LDST128_ABS_LO12_NC: 442 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 443 checkAlignment(loc, val, 16, rel); 444 or32AArch64Imm(loc, getBits(val, 4, 11)); 445 break; 446 case R_AARCH64_LD64_GOTPAGE_LO15: 447 checkAlignment(loc, val, 8, rel); 448 or32AArch64Imm(loc, getBits(val, 3, 14)); 449 break; 450 case R_AARCH64_MOVW_UABS_G0: 451 checkUInt(loc, val, 16, rel); 452 [[fallthrough]]; 453 case R_AARCH64_MOVW_UABS_G0_NC: 454 or32le(loc, (val & 0xFFFF) << 5); 455 break; 456 case R_AARCH64_MOVW_UABS_G1: 457 checkUInt(loc, val, 32, rel); 458 [[fallthrough]]; 459 case R_AARCH64_MOVW_UABS_G1_NC: 460 or32le(loc, (val & 0xFFFF0000) >> 11); 461 break; 462 case R_AARCH64_MOVW_UABS_G2: 463 checkUInt(loc, val, 48, rel); 464 [[fallthrough]]; 465 case R_AARCH64_MOVW_UABS_G2_NC: 466 or32le(loc, (val & 0xFFFF00000000) >> 27); 467 break; 468 case R_AARCH64_MOVW_UABS_G3: 469 or32le(loc, (val & 0xFFFF000000000000) >> 43); 470 break; 471 case R_AARCH64_MOVW_PREL_G0: 472 case R_AARCH64_MOVW_SABS_G0: 473 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 474 checkInt(loc, val, 17, rel); 475 [[fallthrough]]; 476 case R_AARCH64_MOVW_PREL_G0_NC: 477 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 478 writeSMovWImm(loc, val); 479 break; 480 case R_AARCH64_MOVW_PREL_G1: 481 case R_AARCH64_MOVW_SABS_G1: 482 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 483 checkInt(loc, val, 33, rel); 484 [[fallthrough]]; 485 case R_AARCH64_MOVW_PREL_G1_NC: 486 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 487 writeSMovWImm(loc, val >> 16); 488 break; 489 case R_AARCH64_MOVW_PREL_G2: 490 case R_AARCH64_MOVW_SABS_G2: 491 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 492 checkInt(loc, val, 49, rel); 493 [[fallthrough]]; 494 case R_AARCH64_MOVW_PREL_G2_NC: 495 writeSMovWImm(loc, val >> 32); 496 break; 497 case R_AARCH64_MOVW_PREL_G3: 498 writeSMovWImm(loc, val >> 48); 499 break; 500 case R_AARCH64_TSTBR14: 501 checkInt(loc, val, 16, rel); 502 or32le(loc, (val & 0xFFFC) << 3); 503 break; 504 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 505 checkUInt(loc, val, 24, rel); 506 or32AArch64Imm(loc, val >> 12); 507 break; 508 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 509 case R_AARCH64_TLSDESC_ADD_LO12: 510 or32AArch64Imm(loc, val); 511 break; 512 case R_AARCH64_TLSDESC: 513 // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word. 514 write64(loc + 8, val); 515 break; 516 default: 517 llvm_unreachable("unknown relocation"); 518 } 519 } 520 521 void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, 522 uint64_t val) const { 523 // TLSDESC Global-Dynamic relocation are in the form: 524 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 525 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 526 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 527 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 528 // blr x1 529 // And it can optimized to: 530 // movz x0, #0x0, lsl #16 531 // movk x0, #0x10 532 // nop 533 // nop 534 checkUInt(loc, val, 32, rel); 535 536 switch (rel.type) { 537 case R_AARCH64_TLSDESC_ADD_LO12: 538 case R_AARCH64_TLSDESC_CALL: 539 write32le(loc, 0xd503201f); // nop 540 return; 541 case R_AARCH64_TLSDESC_ADR_PAGE21: 542 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz 543 return; 544 case R_AARCH64_TLSDESC_LD64_LO12: 545 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk 546 return; 547 default: 548 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 549 } 550 } 551 552 void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, 553 uint64_t val) const { 554 // TLSDESC Global-Dynamic relocation are in the form: 555 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 556 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 557 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 558 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 559 // blr x1 560 // And it can optimized to: 561 // adrp x0, :gottprel:v 562 // ldr x0, [x0, :gottprel_lo12:v] 563 // nop 564 // nop 565 566 switch (rel.type) { 567 case R_AARCH64_TLSDESC_ADD_LO12: 568 case R_AARCH64_TLSDESC_CALL: 569 write32le(loc, 0xd503201f); // nop 570 break; 571 case R_AARCH64_TLSDESC_ADR_PAGE21: 572 write32le(loc, 0x90000000); // adrp 573 relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val); 574 break; 575 case R_AARCH64_TLSDESC_LD64_LO12: 576 write32le(loc, 0xf9400000); // ldr 577 relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val); 578 break; 579 default: 580 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 581 } 582 } 583 584 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, 585 uint64_t val) const { 586 checkUInt(loc, val, 32, rel); 587 588 if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 589 // Generate MOVZ. 590 uint32_t regNo = read32le(loc) & 0x1f; 591 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5)); 592 return; 593 } 594 if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 595 // Generate MOVK. 596 uint32_t regNo = read32le(loc) & 0x1f; 597 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5)); 598 return; 599 } 600 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 601 } 602 603 AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) { 604 if (!config->relax) 605 return; 606 // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC 607 // always appear in pairs. 608 size_t i = 0; 609 const size_t size = relocs.size(); 610 for (; i != size; ++i) { 611 if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) { 612 if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) { 613 ++i; 614 continue; 615 } 616 break; 617 } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) { 618 break; 619 } 620 } 621 safeToRelaxAdrpLdr = i == size; 622 } 623 624 bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, 625 const Relocation &addRel, uint64_t secAddr, 626 uint8_t *buf) const { 627 // When the address of sym is within the range of ADR then 628 // we may relax 629 // ADRP xn, sym 630 // ADD xn, xn, :lo12: sym 631 // to 632 // NOP 633 // ADR xn, sym 634 if (!config->relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 || 635 addRel.type != R_AARCH64_ADD_ABS_LO12_NC) 636 return false; 637 // Check if the relocations apply to consecutive instructions. 638 if (adrpRel.offset + 4 != addRel.offset) 639 return false; 640 if (adrpRel.sym != addRel.sym) 641 return false; 642 if (adrpRel.addend != 0 || addRel.addend != 0) 643 return false; 644 645 uint32_t adrpInstr = read32le(buf + adrpRel.offset); 646 uint32_t addInstr = read32le(buf + addRel.offset); 647 // Check if the first instruction is ADRP and the second instruction is ADD. 648 if ((adrpInstr & 0x9f000000) != 0x90000000 || 649 (addInstr & 0xffc00000) != 0x91000000) 650 return false; 651 uint32_t adrpDestReg = adrpInstr & 0x1f; 652 uint32_t addDestReg = addInstr & 0x1f; 653 uint32_t addSrcReg = (addInstr >> 5) & 0x1f; 654 if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg) 655 return false; 656 657 Symbol &sym = *adrpRel.sym; 658 // Check if the address difference is within 1MiB range. 659 int64_t val = sym.getVA() - (secAddr + addRel.offset); 660 if (val < -1024 * 1024 || val >= 1024 * 1024) 661 return false; 662 663 Relocation adrRel = {R_ABS, R_AARCH64_ADR_PREL_LO21, addRel.offset, 664 /*addend=*/0, &sym}; 665 // nop 666 write32le(buf + adrpRel.offset, 0xd503201f); 667 // adr x_<dest_reg> 668 write32le(buf + adrRel.offset, 0x10000000 | adrpDestReg); 669 target->relocate(buf + adrRel.offset, adrRel, val); 670 return true; 671 } 672 673 bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, 674 const Relocation &ldrRel, uint64_t secAddr, 675 uint8_t *buf) const { 676 if (!safeToRelaxAdrpLdr) 677 return false; 678 679 // When the definition of sym is not preemptible then we may 680 // be able to relax 681 // ADRP xn, :got: sym 682 // LDR xn, [ xn :got_lo12: sym] 683 // to 684 // ADRP xn, sym 685 // ADD xn, xn, :lo_12: sym 686 687 if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE || 688 ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC) 689 return false; 690 // Check if the relocations apply to consecutive instructions. 691 if (adrpRel.offset + 4 != ldrRel.offset) 692 return false; 693 // Check if the relocations reference the same symbol and 694 // skip undefined, preemptible and STT_GNU_IFUNC symbols. 695 if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() || 696 adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc()) 697 return false; 698 // Check if the addends of the both relocations are zero. 699 if (adrpRel.addend != 0 || ldrRel.addend != 0) 700 return false; 701 uint32_t adrpInstr = read32le(buf + adrpRel.offset); 702 uint32_t ldrInstr = read32le(buf + ldrRel.offset); 703 // Check if the first instruction is ADRP and the second instruction is LDR. 704 if ((adrpInstr & 0x9f000000) != 0x90000000 || 705 (ldrInstr & 0x3b000000) != 0x39000000) 706 return false; 707 // Check the value of the sf bit. 708 if (!(ldrInstr >> 31)) 709 return false; 710 uint32_t adrpDestReg = adrpInstr & 0x1f; 711 uint32_t ldrDestReg = ldrInstr & 0x1f; 712 uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f; 713 // Check if ADPR and LDR use the same register. 714 if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg) 715 return false; 716 717 Symbol &sym = *adrpRel.sym; 718 // GOT references to absolute symbols can't be relaxed to use ADRP/ADD in 719 // position-independent code because these instructions produce a relative 720 // address. 721 if (config->isPic && !cast<Defined>(sym).section) 722 return false; 723 // Check if the address difference is within 4GB range. 724 int64_t val = 725 getAArch64Page(sym.getVA()) - getAArch64Page(secAddr + adrpRel.offset); 726 if (val != llvm::SignExtend64(val, 33)) 727 return false; 728 729 Relocation adrpSymRel = {R_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21, 730 adrpRel.offset, /*addend=*/0, &sym}; 731 Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset, 732 /*addend=*/0, &sym}; 733 734 // adrp x_<dest_reg> 735 write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg); 736 // add x_<dest reg>, x_<dest reg> 737 write32le(buf + addRel.offset, 0x91000000 | adrpDestReg | (adrpDestReg << 5)); 738 739 target->relocate(buf + adrpSymRel.offset, adrpSymRel, 740 SignExtend64(getAArch64Page(sym.getVA()) - 741 getAArch64Page(secAddr + adrpSymRel.offset), 742 64)); 743 target->relocate(buf + addRel.offset, addRel, SignExtend64(sym.getVA(), 64)); 744 tryRelaxAdrpAdd(adrpSymRel, addRel, secAddr, buf); 745 return true; 746 } 747 748 void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { 749 uint64_t secAddr = sec.getOutputSection()->addr; 750 if (auto *s = dyn_cast<InputSection>(&sec)) 751 secAddr += s->outSecOff; 752 AArch64Relaxer relaxer(sec.relocs()); 753 for (size_t i = 0, size = sec.relocs().size(); i != size; ++i) { 754 const Relocation &rel = sec.relocs()[i]; 755 uint8_t *loc = buf + rel.offset; 756 const uint64_t val = 757 sec.getRelocTargetVA(sec.file, rel.type, rel.addend, 758 secAddr + rel.offset, *rel.sym, rel.expr); 759 switch (rel.expr) { 760 case R_AARCH64_GOT_PAGE_PC: 761 if (i + 1 < size && 762 relaxer.tryRelaxAdrpLdr(rel, sec.relocs()[i + 1], secAddr, buf)) { 763 ++i; 764 continue; 765 } 766 break; 767 case R_AARCH64_PAGE_PC: 768 if (i + 1 < size && 769 relaxer.tryRelaxAdrpAdd(rel, sec.relocs()[i + 1], secAddr, buf)) { 770 ++i; 771 continue; 772 } 773 break; 774 case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC: 775 case R_RELAX_TLS_GD_TO_IE_ABS: 776 relaxTlsGdToIe(loc, rel, val); 777 continue; 778 case R_RELAX_TLS_GD_TO_LE: 779 relaxTlsGdToLe(loc, rel, val); 780 continue; 781 case R_RELAX_TLS_IE_TO_LE: 782 relaxTlsIeToLe(loc, rel, val); 783 continue; 784 default: 785 break; 786 } 787 relocate(loc, rel, val); 788 } 789 } 790 791 // AArch64 may use security features in variant PLT sequences. These are: 792 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target 793 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used 794 // in the variant Plt sequences are encoded in the Hint space so they can be 795 // deployed on older architectures, which treat the instructions as a nop. 796 // PAC and BTI can be combined leading to the following combinations: 797 // writePltHeader 798 // writePltHeaderBti (no PAC Header needed) 799 // writePlt 800 // writePltBti (BTI only) 801 // writePltPac (PAC only) 802 // writePltBtiPac (BTI and PAC) 803 // 804 // When PAC is enabled the dynamic loader encrypts the address that it places 805 // in the .got.plt using the pacia1716 instruction which encrypts the value in 806 // x17 using the modifier in x16. The static linker places autia1716 before the 807 // indirect branch to x17 to authenticate the address in x17 with the modifier 808 // in x16. This makes it more difficult for an attacker to modify the value in 809 // the .got.plt. 810 // 811 // When BTI is enabled all indirect branches must land on a bti instruction. 812 // The static linker must place a bti instruction at the start of any PLT entry 813 // that may be the target of an indirect branch. As the PLT entries call the 814 // lazy resolver indirectly this must have a bti instruction at start. In 815 // general a bti instruction is not needed for a PLT entry as indirect calls 816 // are resolved to the function address and not the PLT entry for the function. 817 // There are a small number of cases where the PLT address can escape, such as 818 // taking the address of a function or ifunc via a non got-generating 819 // relocation, and a shared library refers to that symbol. 820 // 821 // We use the bti c variant of the instruction which permits indirect branches 822 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI 823 // guarantees that all indirect branches from code requiring BTI protection 824 // will go via x16/x17 825 826 namespace { 827 class AArch64BtiPac final : public AArch64 { 828 public: 829 AArch64BtiPac(); 830 void writePltHeader(uint8_t *buf) const override; 831 void writePlt(uint8_t *buf, const Symbol &sym, 832 uint64_t pltEntryAddr) const override; 833 834 private: 835 bool btiHeader; // bti instruction needed in PLT Header and Entry 836 bool pacEntry; // autia1716 instruction needed in PLT Entry 837 }; 838 } // namespace 839 840 AArch64BtiPac::AArch64BtiPac() { 841 btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI); 842 // A BTI (Branch Target Indicator) Plt Entry is only required if the 843 // address of the PLT entry can be taken by the program, which permits an 844 // indirect jump to the PLT entry. This can happen when the address 845 // of the PLT entry for a function is canonicalised due to the address of 846 // the function in an executable being taken by a shared library, or 847 // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating 848 // relocations. 849 // The PAC PLT entries require dynamic loader support and this isn't known 850 // from properties in the objects, so we use the command line flag. 851 pacEntry = config->zPacPlt; 852 853 if (btiHeader || pacEntry) { 854 pltEntrySize = 24; 855 ipltEntrySize = 24; 856 } 857 } 858 859 void AArch64BtiPac::writePltHeader(uint8_t *buf) const { 860 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 861 const uint8_t pltData[] = { 862 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 863 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[2])) 864 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[2]))] 865 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.got.plt[2])) 866 0x20, 0x02, 0x1f, 0xd6, // br x17 867 0x1f, 0x20, 0x03, 0xd5, // nop 868 0x1f, 0x20, 0x03, 0xd5 // nop 869 }; 870 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 871 872 uint64_t got = in.gotPlt->getVA(); 873 uint64_t plt = in.plt->getVA(); 874 875 if (btiHeader) { 876 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C 877 // instruction. 878 memcpy(buf, btiData, sizeof(btiData)); 879 buf += sizeof(btiData); 880 plt += sizeof(btiData); 881 } 882 memcpy(buf, pltData, sizeof(pltData)); 883 884 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 885 getAArch64Page(got + 16) - getAArch64Page(plt + 8)); 886 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 887 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 888 if (!btiHeader) 889 // We didn't add the BTI c instruction so round out size with NOP. 890 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData)); 891 } 892 893 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym, 894 uint64_t pltEntryAddr) const { 895 // The PLT entry is of the form: 896 // [btiData] addrInst (pacBr | stdBr) [nopData] 897 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 898 const uint8_t addrInst[] = { 899 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.got.plt[n])) 900 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.got.plt[n]))] 901 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.got.plt[n])) 902 }; 903 const uint8_t pacBr[] = { 904 0x9f, 0x21, 0x03, 0xd5, // autia1716 905 0x20, 0x02, 0x1f, 0xd6 // br x17 906 }; 907 const uint8_t stdBr[] = { 908 0x20, 0x02, 0x1f, 0xd6, // br x17 909 0x1f, 0x20, 0x03, 0xd5 // nop 910 }; 911 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 912 913 // NEEDS_COPY indicates a non-ifunc canonical PLT entry whose address may 914 // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its 915 // address may escape if referenced by a direct relocation. If relative 916 // vtables are used then if the vtable is in a shared object the offsets will 917 // be to the PLT entry. The condition is conservative. 918 bool hasBti = btiHeader && 919 (sym.hasFlag(NEEDS_COPY) || sym.isInIplt || sym.thunkAccessed); 920 if (hasBti) { 921 memcpy(buf, btiData, sizeof(btiData)); 922 buf += sizeof(btiData); 923 pltEntryAddr += sizeof(btiData); 924 } 925 926 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 927 memcpy(buf, addrInst, sizeof(addrInst)); 928 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21, 929 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 930 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 931 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 932 933 if (pacEntry) 934 memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr)); 935 else 936 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr)); 937 if (!hasBti) 938 // We didn't add the BTI c instruction so round out size with NOP. 939 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData)); 940 } 941 942 static TargetInfo *getTargetInfo() { 943 if ((config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI) || 944 config->zPacPlt) { 945 static AArch64BtiPac t; 946 return &t; 947 } 948 static AArch64 t; 949 return &t; 950 } 951 952 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); } 953