1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Symbols.h" 10 #include "SyntheticSections.h" 11 #include "Target.h" 12 #include "Thunks.h" 13 #include "lld/Common/ErrorHandler.h" 14 #include "llvm/Object/ELF.h" 15 #include "llvm/Support/Endian.h" 16 17 using namespace llvm; 18 using namespace llvm::support::endian; 19 using namespace llvm::ELF; 20 using namespace lld; 21 using namespace lld::elf; 22 23 // Page(Expr) is the page address of the expression Expr, defined 24 // as (Expr & ~0xFFF). (This applies even if the machine page size 25 // supported by the platform has a different value.) 26 uint64_t elf::getAArch64Page(uint64_t expr) { 27 return expr & ~static_cast<uint64_t>(0xFFF); 28 } 29 30 namespace { 31 class AArch64 : public TargetInfo { 32 public: 33 AArch64(); 34 RelExpr getRelExpr(RelType type, const Symbol &s, 35 const uint8_t *loc) const override; 36 RelType getDynRel(RelType type) const override; 37 int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; 38 void writeGotPlt(uint8_t *buf, const Symbol &s) const override; 39 void writePltHeader(uint8_t *buf) const override; 40 void writePlt(uint8_t *buf, const Symbol &sym, 41 uint64_t pltEntryAddr) const override; 42 bool needsThunk(RelExpr expr, RelType type, const InputFile *file, 43 uint64_t branchAddr, const Symbol &s, 44 int64_t a) const override; 45 uint32_t getThunkSectionSpacing() const override; 46 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; 47 bool usesOnlyLowPageBits(RelType type) const override; 48 void relocate(uint8_t *loc, const Relocation &rel, 49 uint64_t val) const override; 50 RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; 51 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, 52 uint64_t val) const override; 53 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, 54 uint64_t val) const override; 55 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, 56 uint64_t val) const override; 57 }; 58 } // namespace 59 60 AArch64::AArch64() { 61 copyRel = R_AARCH64_COPY; 62 relativeRel = R_AARCH64_RELATIVE; 63 iRelativeRel = R_AARCH64_IRELATIVE; 64 gotRel = R_AARCH64_GLOB_DAT; 65 pltRel = R_AARCH64_JUMP_SLOT; 66 symbolicRel = R_AARCH64_ABS64; 67 tlsDescRel = R_AARCH64_TLSDESC; 68 tlsGotRel = R_AARCH64_TLS_TPREL64; 69 pltHeaderSize = 32; 70 pltEntrySize = 16; 71 ipltEntrySize = 16; 72 defaultMaxPageSize = 65536; 73 74 // Align to the 2 MiB page size (known as a superpage or huge page). 75 // FreeBSD automatically promotes 2 MiB-aligned allocations. 76 defaultImageBase = 0x200000; 77 78 needsThunks = true; 79 } 80 81 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, 82 const uint8_t *loc) const { 83 switch (type) { 84 case R_AARCH64_ABS16: 85 case R_AARCH64_ABS32: 86 case R_AARCH64_ABS64: 87 case R_AARCH64_ADD_ABS_LO12_NC: 88 case R_AARCH64_LDST128_ABS_LO12_NC: 89 case R_AARCH64_LDST16_ABS_LO12_NC: 90 case R_AARCH64_LDST32_ABS_LO12_NC: 91 case R_AARCH64_LDST64_ABS_LO12_NC: 92 case R_AARCH64_LDST8_ABS_LO12_NC: 93 case R_AARCH64_MOVW_SABS_G0: 94 case R_AARCH64_MOVW_SABS_G1: 95 case R_AARCH64_MOVW_SABS_G2: 96 case R_AARCH64_MOVW_UABS_G0: 97 case R_AARCH64_MOVW_UABS_G0_NC: 98 case R_AARCH64_MOVW_UABS_G1: 99 case R_AARCH64_MOVW_UABS_G1_NC: 100 case R_AARCH64_MOVW_UABS_G2: 101 case R_AARCH64_MOVW_UABS_G2_NC: 102 case R_AARCH64_MOVW_UABS_G3: 103 return R_ABS; 104 case R_AARCH64_TLSDESC_ADR_PAGE21: 105 return R_AARCH64_TLSDESC_PAGE; 106 case R_AARCH64_TLSDESC_LD64_LO12: 107 case R_AARCH64_TLSDESC_ADD_LO12: 108 return R_TLSDESC; 109 case R_AARCH64_TLSDESC_CALL: 110 return R_TLSDESC_CALL; 111 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 112 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 113 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 114 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 115 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 116 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 117 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 118 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 119 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 120 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 121 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 122 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 123 return R_TPREL; 124 case R_AARCH64_CALL26: 125 case R_AARCH64_CONDBR19: 126 case R_AARCH64_JUMP26: 127 case R_AARCH64_TSTBR14: 128 case R_AARCH64_PLT32: 129 return R_PLT_PC; 130 case R_AARCH64_PREL16: 131 case R_AARCH64_PREL32: 132 case R_AARCH64_PREL64: 133 case R_AARCH64_ADR_PREL_LO21: 134 case R_AARCH64_LD_PREL_LO19: 135 case R_AARCH64_MOVW_PREL_G0: 136 case R_AARCH64_MOVW_PREL_G0_NC: 137 case R_AARCH64_MOVW_PREL_G1: 138 case R_AARCH64_MOVW_PREL_G1_NC: 139 case R_AARCH64_MOVW_PREL_G2: 140 case R_AARCH64_MOVW_PREL_G2_NC: 141 case R_AARCH64_MOVW_PREL_G3: 142 return R_PC; 143 case R_AARCH64_ADR_PREL_PG_HI21: 144 case R_AARCH64_ADR_PREL_PG_HI21_NC: 145 return R_AARCH64_PAGE_PC; 146 case R_AARCH64_LD64_GOT_LO12_NC: 147 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 148 return R_GOT; 149 case R_AARCH64_LD64_GOTPAGE_LO15: 150 return R_AARCH64_GOT_PAGE; 151 case R_AARCH64_ADR_GOT_PAGE: 152 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 153 return R_AARCH64_GOT_PAGE_PC; 154 case R_AARCH64_NONE: 155 return R_NONE; 156 default: 157 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + 158 ") against symbol " + toString(s)); 159 return R_NONE; 160 } 161 } 162 163 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const { 164 if (expr == R_RELAX_TLS_GD_TO_IE) { 165 if (type == R_AARCH64_TLSDESC_ADR_PAGE21) 166 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; 167 return R_RELAX_TLS_GD_TO_IE_ABS; 168 } 169 return expr; 170 } 171 172 bool AArch64::usesOnlyLowPageBits(RelType type) const { 173 switch (type) { 174 default: 175 return false; 176 case R_AARCH64_ADD_ABS_LO12_NC: 177 case R_AARCH64_LD64_GOT_LO12_NC: 178 case R_AARCH64_LDST128_ABS_LO12_NC: 179 case R_AARCH64_LDST16_ABS_LO12_NC: 180 case R_AARCH64_LDST32_ABS_LO12_NC: 181 case R_AARCH64_LDST64_ABS_LO12_NC: 182 case R_AARCH64_LDST8_ABS_LO12_NC: 183 case R_AARCH64_TLSDESC_ADD_LO12: 184 case R_AARCH64_TLSDESC_LD64_LO12: 185 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 186 return true; 187 } 188 } 189 190 RelType AArch64::getDynRel(RelType type) const { 191 if (type == R_AARCH64_ABS64) 192 return type; 193 return R_AARCH64_NONE; 194 } 195 196 int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const { 197 switch (type) { 198 case R_AARCH64_TLSDESC: 199 return read64(buf + 8); 200 case R_AARCH64_NONE: 201 return 0; 202 case R_AARCH64_PREL32: 203 return SignExtend64<32>(read32(buf)); 204 case R_AARCH64_ABS64: 205 case R_AARCH64_PREL64: 206 return read64(buf); 207 default: 208 internalLinkerError(getErrorLocation(buf), 209 "cannot read addend for relocation " + toString(type)); 210 return 0; 211 } 212 } 213 214 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const { 215 write64(buf, in.plt->getVA()); 216 } 217 218 void AArch64::writePltHeader(uint8_t *buf) const { 219 const uint8_t pltData[] = { 220 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 221 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 222 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 223 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 224 0x20, 0x02, 0x1f, 0xd6, // br x17 225 0x1f, 0x20, 0x03, 0xd5, // nop 226 0x1f, 0x20, 0x03, 0xd5, // nop 227 0x1f, 0x20, 0x03, 0xd5 // nop 228 }; 229 memcpy(buf, pltData, sizeof(pltData)); 230 231 uint64_t got = in.gotPlt->getVA(); 232 uint64_t plt = in.plt->getVA(); 233 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 234 getAArch64Page(got + 16) - getAArch64Page(plt + 4)); 235 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 236 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 237 } 238 239 void AArch64::writePlt(uint8_t *buf, const Symbol &sym, 240 uint64_t pltEntryAddr) const { 241 const uint8_t inst[] = { 242 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 243 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 244 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 245 0x20, 0x02, 0x1f, 0xd6 // br x17 246 }; 247 memcpy(buf, inst, sizeof(inst)); 248 249 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 250 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21, 251 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 252 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 253 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 254 } 255 256 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file, 257 uint64_t branchAddr, const Symbol &s, 258 int64_t a) const { 259 // If s is an undefined weak symbol and does not have a PLT entry then it 260 // will be resolved as a branch to the next instruction. 261 if (s.isUndefWeak() && !s.isInPlt()) 262 return false; 263 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 264 // only permits range extension thunks for R_AARCH64_CALL26 and 265 // R_AARCH64_JUMP26 relocation types. 266 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && 267 type != R_AARCH64_PLT32) 268 return false; 269 uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a); 270 return !inBranchRange(type, branchAddr, dst); 271 } 272 273 uint32_t AArch64::getThunkSectionSpacing() const { 274 // See comment in Arch/ARM.cpp for a more detailed explanation of 275 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 276 // Thunk have a range of +/- 128 MiB 277 return (128 * 1024 * 1024) - 0x30000; 278 } 279 280 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { 281 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && 282 type != R_AARCH64_PLT32) 283 return true; 284 // The AArch64 call and unconditional branch instructions have a range of 285 // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB. 286 uint64_t range = 287 type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024); 288 if (dst > src) { 289 // Immediate of branch is signed. 290 range -= 4; 291 return dst - src <= range; 292 } 293 return src - dst <= range; 294 } 295 296 static void write32AArch64Addr(uint8_t *l, uint64_t imm) { 297 uint32_t immLo = (imm & 0x3) << 29; 298 uint32_t immHi = (imm & 0x1FFFFC) << 3; 299 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3); 300 write32le(l, (read32le(l) & ~mask) | immLo | immHi); 301 } 302 303 // Return the bits [Start, End] from Val shifted Start bits. 304 // For instance, getBits(0xF0, 4, 8) returns 0xF. 305 static uint64_t getBits(uint64_t val, int start, int end) { 306 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1; 307 return (val >> start) & mask; 308 } 309 310 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); } 311 312 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 313 static void or32AArch64Imm(uint8_t *l, uint64_t imm) { 314 or32le(l, (imm & 0xFFF) << 10); 315 } 316 317 // Update the immediate field in an AArch64 movk, movn or movz instruction 318 // for a signed relocation, and update the opcode of a movn or movz instruction 319 // to match the sign of the operand. 320 static void writeSMovWImm(uint8_t *loc, uint32_t imm) { 321 uint32_t inst = read32le(loc); 322 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. 323 if (!(inst & (1 << 29))) { 324 // movn or movz. 325 if (imm & 0x10000) { 326 // Change opcode to movn, which takes an inverted operand. 327 imm ^= 0xFFFF; 328 inst &= ~(1 << 30); 329 } else { 330 // Change opcode to movz. 331 inst |= 1 << 30; 332 } 333 } 334 write32le(loc, inst | ((imm & 0xFFFF) << 5)); 335 } 336 337 void AArch64::relocate(uint8_t *loc, const Relocation &rel, 338 uint64_t val) const { 339 switch (rel.type) { 340 case R_AARCH64_ABS16: 341 case R_AARCH64_PREL16: 342 checkIntUInt(loc, val, 16, rel); 343 write16(loc, val); 344 break; 345 case R_AARCH64_ABS32: 346 case R_AARCH64_PREL32: 347 checkIntUInt(loc, val, 32, rel); 348 write32(loc, val); 349 break; 350 case R_AARCH64_PLT32: 351 checkInt(loc, val, 32, rel); 352 write32(loc, val); 353 break; 354 case R_AARCH64_ABS64: 355 case R_AARCH64_PREL64: 356 write64(loc, val); 357 break; 358 case R_AARCH64_ADD_ABS_LO12_NC: 359 or32AArch64Imm(loc, val); 360 break; 361 case R_AARCH64_ADR_GOT_PAGE: 362 case R_AARCH64_ADR_PREL_PG_HI21: 363 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 364 case R_AARCH64_TLSDESC_ADR_PAGE21: 365 checkInt(loc, val, 33, rel); 366 LLVM_FALLTHROUGH; 367 case R_AARCH64_ADR_PREL_PG_HI21_NC: 368 write32AArch64Addr(loc, val >> 12); 369 break; 370 case R_AARCH64_ADR_PREL_LO21: 371 checkInt(loc, val, 21, rel); 372 write32AArch64Addr(loc, val); 373 break; 374 case R_AARCH64_JUMP26: 375 // Normally we would just write the bits of the immediate field, however 376 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 377 // we want to replace a non-branch instruction with a branch immediate 378 // instruction. By writing all the bits of the instruction including the 379 // opcode and the immediate (0 001 | 01 imm26) we can do this 380 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 381 // the instruction we want to patch. 382 write32le(loc, 0x14000000); 383 LLVM_FALLTHROUGH; 384 case R_AARCH64_CALL26: 385 checkInt(loc, val, 28, rel); 386 or32le(loc, (val & 0x0FFFFFFC) >> 2); 387 break; 388 case R_AARCH64_CONDBR19: 389 case R_AARCH64_LD_PREL_LO19: 390 checkAlignment(loc, val, 4, rel); 391 checkInt(loc, val, 21, rel); 392 or32le(loc, (val & 0x1FFFFC) << 3); 393 break; 394 case R_AARCH64_LDST8_ABS_LO12_NC: 395 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 396 or32AArch64Imm(loc, getBits(val, 0, 11)); 397 break; 398 case R_AARCH64_LDST16_ABS_LO12_NC: 399 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 400 checkAlignment(loc, val, 2, rel); 401 or32AArch64Imm(loc, getBits(val, 1, 11)); 402 break; 403 case R_AARCH64_LDST32_ABS_LO12_NC: 404 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 405 checkAlignment(loc, val, 4, rel); 406 or32AArch64Imm(loc, getBits(val, 2, 11)); 407 break; 408 case R_AARCH64_LDST64_ABS_LO12_NC: 409 case R_AARCH64_LD64_GOT_LO12_NC: 410 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 411 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 412 case R_AARCH64_TLSDESC_LD64_LO12: 413 checkAlignment(loc, val, 8, rel); 414 or32AArch64Imm(loc, getBits(val, 3, 11)); 415 break; 416 case R_AARCH64_LDST128_ABS_LO12_NC: 417 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 418 checkAlignment(loc, val, 16, rel); 419 or32AArch64Imm(loc, getBits(val, 4, 11)); 420 break; 421 case R_AARCH64_LD64_GOTPAGE_LO15: 422 checkAlignment(loc, val, 8, rel); 423 or32AArch64Imm(loc, getBits(val, 3, 14)); 424 break; 425 case R_AARCH64_MOVW_UABS_G0: 426 checkUInt(loc, val, 16, rel); 427 LLVM_FALLTHROUGH; 428 case R_AARCH64_MOVW_UABS_G0_NC: 429 or32le(loc, (val & 0xFFFF) << 5); 430 break; 431 case R_AARCH64_MOVW_UABS_G1: 432 checkUInt(loc, val, 32, rel); 433 LLVM_FALLTHROUGH; 434 case R_AARCH64_MOVW_UABS_G1_NC: 435 or32le(loc, (val & 0xFFFF0000) >> 11); 436 break; 437 case R_AARCH64_MOVW_UABS_G2: 438 checkUInt(loc, val, 48, rel); 439 LLVM_FALLTHROUGH; 440 case R_AARCH64_MOVW_UABS_G2_NC: 441 or32le(loc, (val & 0xFFFF00000000) >> 27); 442 break; 443 case R_AARCH64_MOVW_UABS_G3: 444 or32le(loc, (val & 0xFFFF000000000000) >> 43); 445 break; 446 case R_AARCH64_MOVW_PREL_G0: 447 case R_AARCH64_MOVW_SABS_G0: 448 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 449 checkInt(loc, val, 17, rel); 450 LLVM_FALLTHROUGH; 451 case R_AARCH64_MOVW_PREL_G0_NC: 452 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 453 writeSMovWImm(loc, val); 454 break; 455 case R_AARCH64_MOVW_PREL_G1: 456 case R_AARCH64_MOVW_SABS_G1: 457 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 458 checkInt(loc, val, 33, rel); 459 LLVM_FALLTHROUGH; 460 case R_AARCH64_MOVW_PREL_G1_NC: 461 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 462 writeSMovWImm(loc, val >> 16); 463 break; 464 case R_AARCH64_MOVW_PREL_G2: 465 case R_AARCH64_MOVW_SABS_G2: 466 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 467 checkInt(loc, val, 49, rel); 468 LLVM_FALLTHROUGH; 469 case R_AARCH64_MOVW_PREL_G2_NC: 470 writeSMovWImm(loc, val >> 32); 471 break; 472 case R_AARCH64_MOVW_PREL_G3: 473 writeSMovWImm(loc, val >> 48); 474 break; 475 case R_AARCH64_TSTBR14: 476 checkInt(loc, val, 16, rel); 477 or32le(loc, (val & 0xFFFC) << 3); 478 break; 479 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 480 checkUInt(loc, val, 24, rel); 481 or32AArch64Imm(loc, val >> 12); 482 break; 483 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 484 case R_AARCH64_TLSDESC_ADD_LO12: 485 or32AArch64Imm(loc, val); 486 break; 487 case R_AARCH64_TLSDESC: 488 // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word. 489 write64(loc + 8, val); 490 break; 491 default: 492 llvm_unreachable("unknown relocation"); 493 } 494 } 495 496 void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, 497 uint64_t val) const { 498 // TLSDESC Global-Dynamic relocation are in the form: 499 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 500 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 501 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 502 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 503 // blr x1 504 // And it can optimized to: 505 // movz x0, #0x0, lsl #16 506 // movk x0, #0x10 507 // nop 508 // nop 509 checkUInt(loc, val, 32, rel); 510 511 switch (rel.type) { 512 case R_AARCH64_TLSDESC_ADD_LO12: 513 case R_AARCH64_TLSDESC_CALL: 514 write32le(loc, 0xd503201f); // nop 515 return; 516 case R_AARCH64_TLSDESC_ADR_PAGE21: 517 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz 518 return; 519 case R_AARCH64_TLSDESC_LD64_LO12: 520 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk 521 return; 522 default: 523 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 524 } 525 } 526 527 void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, 528 uint64_t val) const { 529 // TLSDESC Global-Dynamic relocation are in the form: 530 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 531 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 532 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 533 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 534 // blr x1 535 // And it can optimized to: 536 // adrp x0, :gottprel:v 537 // ldr x0, [x0, :gottprel_lo12:v] 538 // nop 539 // nop 540 541 switch (rel.type) { 542 case R_AARCH64_TLSDESC_ADD_LO12: 543 case R_AARCH64_TLSDESC_CALL: 544 write32le(loc, 0xd503201f); // nop 545 break; 546 case R_AARCH64_TLSDESC_ADR_PAGE21: 547 write32le(loc, 0x90000000); // adrp 548 relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val); 549 break; 550 case R_AARCH64_TLSDESC_LD64_LO12: 551 write32le(loc, 0xf9400000); // ldr 552 relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val); 553 break; 554 default: 555 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 556 } 557 } 558 559 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, 560 uint64_t val) const { 561 checkUInt(loc, val, 32, rel); 562 563 if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 564 // Generate MOVZ. 565 uint32_t regNo = read32le(loc) & 0x1f; 566 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5)); 567 return; 568 } 569 if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 570 // Generate MOVK. 571 uint32_t regNo = read32le(loc) & 0x1f; 572 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5)); 573 return; 574 } 575 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 576 } 577 578 AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) { 579 if (!config->relax || config->emachine != EM_AARCH64) { 580 safeToRelaxAdrpLdr = false; 581 return; 582 } 583 // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC 584 // always appear in pairs. 585 size_t i = 0; 586 const size_t size = relocs.size(); 587 for (; i != size; ++i) { 588 if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) { 589 if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) { 590 ++i; 591 continue; 592 } 593 break; 594 } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) { 595 break; 596 } 597 } 598 safeToRelaxAdrpLdr = i == size; 599 } 600 601 bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, 602 const Relocation &addRel, uint64_t secAddr, 603 uint8_t *buf) const { 604 // When the address of sym is within the range of ADR then 605 // we may relax 606 // ADRP xn, sym 607 // ADD xn, xn, :lo12: sym 608 // to 609 // NOP 610 // ADR xn, sym 611 if (!config->relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 || 612 addRel.type != R_AARCH64_ADD_ABS_LO12_NC) 613 return false; 614 // Check if the relocations apply to consecutive instructions. 615 if (adrpRel.offset + 4 != addRel.offset) 616 return false; 617 if (adrpRel.sym != addRel.sym) 618 return false; 619 if (adrpRel.addend != 0 || addRel.addend != 0) 620 return false; 621 622 uint32_t adrpInstr = read32le(buf + adrpRel.offset); 623 uint32_t addInstr = read32le(buf + addRel.offset); 624 // Check if the first instruction is ADRP and the second instruction is ADD. 625 if ((adrpInstr & 0x9f000000) != 0x90000000 || 626 (addInstr & 0xffc00000) != 0x91000000) 627 return false; 628 uint32_t adrpDestReg = adrpInstr & 0x1f; 629 uint32_t addDestReg = addInstr & 0x1f; 630 uint32_t addSrcReg = (addInstr >> 5) & 0x1f; 631 if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg) 632 return false; 633 634 Symbol &sym = *adrpRel.sym; 635 // Check if the address difference is within 1MiB range. 636 int64_t val = sym.getVA() - (secAddr + addRel.offset); 637 if (val < -1024 * 1024 || val >= 1024 * 1024) 638 return false; 639 640 Relocation adrRel = {R_ABS, R_AARCH64_ADR_PREL_LO21, addRel.offset, 641 /*addend=*/0, &sym}; 642 // nop 643 write32le(buf + adrpRel.offset, 0xd503201f); 644 // adr x_<dest_reg> 645 write32le(buf + adrRel.offset, 0x10000000 | adrpDestReg); 646 target->relocate(buf + adrRel.offset, adrRel, val); 647 return true; 648 } 649 650 bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, 651 const Relocation &ldrRel, uint64_t secAddr, 652 uint8_t *buf) const { 653 if (!safeToRelaxAdrpLdr) 654 return false; 655 656 // When the definition of sym is not preemptible then we may 657 // be able to relax 658 // ADRP xn, :got: sym 659 // LDR xn, [ xn :got_lo12: sym] 660 // to 661 // ADRP xn, sym 662 // ADD xn, xn, :lo_12: sym 663 664 if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE || 665 ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC) 666 return false; 667 // Check if the relocations apply to consecutive instructions. 668 if (adrpRel.offset + 4 != ldrRel.offset) 669 return false; 670 // Check if the relocations reference the same symbol and 671 // skip undefined, preemptible and STT_GNU_IFUNC symbols. 672 if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() || 673 adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc()) 674 return false; 675 // Check if the addends of the both relocations are zero. 676 if (adrpRel.addend != 0 || ldrRel.addend != 0) 677 return false; 678 uint32_t adrpInstr = read32le(buf + adrpRel.offset); 679 uint32_t ldrInstr = read32le(buf + ldrRel.offset); 680 // Check if the first instruction is ADRP and the second instruction is LDR. 681 if ((adrpInstr & 0x9f000000) != 0x90000000 || 682 (ldrInstr & 0x3b000000) != 0x39000000) 683 return false; 684 // Check the value of the sf bit. 685 if (!(ldrInstr >> 31)) 686 return false; 687 uint32_t adrpDestReg = adrpInstr & 0x1f; 688 uint32_t ldrDestReg = ldrInstr & 0x1f; 689 uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f; 690 // Check if ADPR and LDR use the same register. 691 if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg) 692 return false; 693 694 Symbol &sym = *adrpRel.sym; 695 // Check if the address difference is within 4GB range. 696 int64_t val = 697 getAArch64Page(sym.getVA()) - getAArch64Page(secAddr + adrpRel.offset); 698 if (val != llvm::SignExtend64(val, 33)) 699 return false; 700 701 Relocation adrpSymRel = {R_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21, 702 adrpRel.offset, /*addend=*/0, &sym}; 703 Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset, 704 /*addend=*/0, &sym}; 705 706 // adrp x_<dest_reg> 707 write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg); 708 // add x_<dest reg>, x_<dest reg> 709 write32le(buf + addRel.offset, 0x91000000 | adrpDestReg | (adrpDestReg << 5)); 710 711 target->relocate(buf + adrpSymRel.offset, adrpSymRel, 712 SignExtend64(getAArch64Page(sym.getVA()) - 713 getAArch64Page(secAddr + adrpSymRel.offset), 714 64)); 715 target->relocate(buf + addRel.offset, addRel, SignExtend64(sym.getVA(), 64)); 716 tryRelaxAdrpAdd(adrpSymRel, addRel, secAddr, buf); 717 return true; 718 } 719 720 // AArch64 may use security features in variant PLT sequences. These are: 721 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target 722 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used 723 // in the variant Plt sequences are encoded in the Hint space so they can be 724 // deployed on older architectures, which treat the instructions as a nop. 725 // PAC and BTI can be combined leading to the following combinations: 726 // writePltHeader 727 // writePltHeaderBti (no PAC Header needed) 728 // writePlt 729 // writePltBti (BTI only) 730 // writePltPac (PAC only) 731 // writePltBtiPac (BTI and PAC) 732 // 733 // When PAC is enabled the dynamic loader encrypts the address that it places 734 // in the .got.plt using the pacia1716 instruction which encrypts the value in 735 // x17 using the modifier in x16. The static linker places autia1716 before the 736 // indirect branch to x17 to authenticate the address in x17 with the modifier 737 // in x16. This makes it more difficult for an attacker to modify the value in 738 // the .got.plt. 739 // 740 // When BTI is enabled all indirect branches must land on a bti instruction. 741 // The static linker must place a bti instruction at the start of any PLT entry 742 // that may be the target of an indirect branch. As the PLT entries call the 743 // lazy resolver indirectly this must have a bti instruction at start. In 744 // general a bti instruction is not needed for a PLT entry as indirect calls 745 // are resolved to the function address and not the PLT entry for the function. 746 // There are a small number of cases where the PLT address can escape, such as 747 // taking the address of a function or ifunc via a non got-generating 748 // relocation, and a shared library refers to that symbol. 749 // 750 // We use the bti c variant of the instruction which permits indirect branches 751 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI 752 // guarantees that all indirect branches from code requiring BTI protection 753 // will go via x16/x17 754 755 namespace { 756 class AArch64BtiPac final : public AArch64 { 757 public: 758 AArch64BtiPac(); 759 void writePltHeader(uint8_t *buf) const override; 760 void writePlt(uint8_t *buf, const Symbol &sym, 761 uint64_t pltEntryAddr) const override; 762 763 private: 764 bool btiHeader; // bti instruction needed in PLT Header and Entry 765 bool pacEntry; // autia1716 instruction needed in PLT Entry 766 }; 767 } // namespace 768 769 AArch64BtiPac::AArch64BtiPac() { 770 btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI); 771 // A BTI (Branch Target Indicator) Plt Entry is only required if the 772 // address of the PLT entry can be taken by the program, which permits an 773 // indirect jump to the PLT entry. This can happen when the address 774 // of the PLT entry for a function is canonicalised due to the address of 775 // the function in an executable being taken by a shared library, or 776 // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating 777 // relocations. 778 // The PAC PLT entries require dynamic loader support and this isn't known 779 // from properties in the objects, so we use the command line flag. 780 pacEntry = config->zPacPlt; 781 782 if (btiHeader || pacEntry) { 783 pltEntrySize = 24; 784 ipltEntrySize = 24; 785 } 786 } 787 788 void AArch64BtiPac::writePltHeader(uint8_t *buf) const { 789 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 790 const uint8_t pltData[] = { 791 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 792 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 793 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 794 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 795 0x20, 0x02, 0x1f, 0xd6, // br x17 796 0x1f, 0x20, 0x03, 0xd5, // nop 797 0x1f, 0x20, 0x03, 0xd5 // nop 798 }; 799 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 800 801 uint64_t got = in.gotPlt->getVA(); 802 uint64_t plt = in.plt->getVA(); 803 804 if (btiHeader) { 805 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C 806 // instruction. 807 memcpy(buf, btiData, sizeof(btiData)); 808 buf += sizeof(btiData); 809 plt += sizeof(btiData); 810 } 811 memcpy(buf, pltData, sizeof(pltData)); 812 813 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 814 getAArch64Page(got + 16) - getAArch64Page(plt + 8)); 815 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 816 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 817 if (!btiHeader) 818 // We didn't add the BTI c instruction so round out size with NOP. 819 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData)); 820 } 821 822 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym, 823 uint64_t pltEntryAddr) const { 824 // The PLT entry is of the form: 825 // [btiData] addrInst (pacBr | stdBr) [nopData] 826 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 827 const uint8_t addrInst[] = { 828 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 829 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 830 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.plt.got[n])) 831 }; 832 const uint8_t pacBr[] = { 833 0x9f, 0x21, 0x03, 0xd5, // autia1716 834 0x20, 0x02, 0x1f, 0xd6 // br x17 835 }; 836 const uint8_t stdBr[] = { 837 0x20, 0x02, 0x1f, 0xd6, // br x17 838 0x1f, 0x20, 0x03, 0xd5 // nop 839 }; 840 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 841 842 // needsCopy indicates a non-ifunc canonical PLT entry whose address may 843 // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its 844 // address may escape if referenced by a direct relocation. The condition is 845 // conservative. 846 bool hasBti = btiHeader && (sym.needsCopy || sym.isInIplt); 847 if (hasBti) { 848 memcpy(buf, btiData, sizeof(btiData)); 849 buf += sizeof(btiData); 850 pltEntryAddr += sizeof(btiData); 851 } 852 853 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 854 memcpy(buf, addrInst, sizeof(addrInst)); 855 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21, 856 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 857 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 858 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 859 860 if (pacEntry) 861 memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr)); 862 else 863 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr)); 864 if (!hasBti) 865 // We didn't add the BTI c instruction so round out size with NOP. 866 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData)); 867 } 868 869 static TargetInfo *getTargetInfo() { 870 if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI | 871 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) { 872 static AArch64BtiPac t; 873 return &t; 874 } 875 static AArch64 t; 876 return &t; 877 } 878 879 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); } 880