1 //===- AArch64.cpp --------------------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Symbols.h" 10 #include "SyntheticSections.h" 11 #include "Target.h" 12 #include "Thunks.h" 13 #include "lld/Common/ErrorHandler.h" 14 #include "llvm/Object/ELF.h" 15 #include "llvm/Support/Endian.h" 16 17 using namespace llvm; 18 using namespace llvm::support::endian; 19 using namespace llvm::ELF; 20 using namespace lld; 21 using namespace lld::elf; 22 23 // Page(Expr) is the page address of the expression Expr, defined 24 // as (Expr & ~0xFFF). (This applies even if the machine page size 25 // supported by the platform has a different value.) 26 uint64_t elf::getAArch64Page(uint64_t expr) { 27 return expr & ~static_cast<uint64_t>(0xFFF); 28 } 29 30 namespace { 31 class AArch64 : public TargetInfo { 32 public: 33 AArch64(); 34 RelExpr getRelExpr(RelType type, const Symbol &s, 35 const uint8_t *loc) const override; 36 RelType getDynRel(RelType type) const override; 37 void writeGotPlt(uint8_t *buf, const Symbol &s) const override; 38 void writePltHeader(uint8_t *buf) const override; 39 void writePlt(uint8_t *buf, const Symbol &sym, 40 uint64_t pltEntryAddr) const override; 41 bool needsThunk(RelExpr expr, RelType type, const InputFile *file, 42 uint64_t branchAddr, const Symbol &s, 43 int64_t a) const override; 44 uint32_t getThunkSectionSpacing() const override; 45 bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override; 46 bool usesOnlyLowPageBits(RelType type) const override; 47 void relocate(uint8_t *loc, const Relocation &rel, 48 uint64_t val) const override; 49 RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; 50 void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, 51 uint64_t val) const override; 52 void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, 53 uint64_t val) const override; 54 void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, 55 uint64_t val) const override; 56 }; 57 } // namespace 58 59 AArch64::AArch64() { 60 copyRel = R_AARCH64_COPY; 61 relativeRel = R_AARCH64_RELATIVE; 62 iRelativeRel = R_AARCH64_IRELATIVE; 63 gotRel = R_AARCH64_GLOB_DAT; 64 noneRel = R_AARCH64_NONE; 65 pltRel = R_AARCH64_JUMP_SLOT; 66 symbolicRel = R_AARCH64_ABS64; 67 tlsDescRel = R_AARCH64_TLSDESC; 68 tlsGotRel = R_AARCH64_TLS_TPREL64; 69 pltHeaderSize = 32; 70 pltEntrySize = 16; 71 ipltEntrySize = 16; 72 defaultMaxPageSize = 65536; 73 gotBaseSymInGotPlt = false; 74 75 // Align to the 2 MiB page size (known as a superpage or huge page). 76 // FreeBSD automatically promotes 2 MiB-aligned allocations. 77 defaultImageBase = 0x200000; 78 79 needsThunks = true; 80 } 81 82 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s, 83 const uint8_t *loc) const { 84 switch (type) { 85 case R_AARCH64_ABS16: 86 case R_AARCH64_ABS32: 87 case R_AARCH64_ABS64: 88 case R_AARCH64_ADD_ABS_LO12_NC: 89 case R_AARCH64_LDST128_ABS_LO12_NC: 90 case R_AARCH64_LDST16_ABS_LO12_NC: 91 case R_AARCH64_LDST32_ABS_LO12_NC: 92 case R_AARCH64_LDST64_ABS_LO12_NC: 93 case R_AARCH64_LDST8_ABS_LO12_NC: 94 case R_AARCH64_MOVW_SABS_G0: 95 case R_AARCH64_MOVW_SABS_G1: 96 case R_AARCH64_MOVW_SABS_G2: 97 case R_AARCH64_MOVW_UABS_G0: 98 case R_AARCH64_MOVW_UABS_G0_NC: 99 case R_AARCH64_MOVW_UABS_G1: 100 case R_AARCH64_MOVW_UABS_G1_NC: 101 case R_AARCH64_MOVW_UABS_G2: 102 case R_AARCH64_MOVW_UABS_G2_NC: 103 case R_AARCH64_MOVW_UABS_G3: 104 return R_ABS; 105 case R_AARCH64_TLSDESC_ADR_PAGE21: 106 return R_AARCH64_TLSDESC_PAGE; 107 case R_AARCH64_TLSDESC_LD64_LO12: 108 case R_AARCH64_TLSDESC_ADD_LO12: 109 return R_TLSDESC; 110 case R_AARCH64_TLSDESC_CALL: 111 return R_TLSDESC_CALL; 112 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 113 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 114 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 115 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 116 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 117 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 118 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 119 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 120 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 121 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 122 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 123 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 124 return R_TPREL; 125 case R_AARCH64_CALL26: 126 case R_AARCH64_CONDBR19: 127 case R_AARCH64_JUMP26: 128 case R_AARCH64_TSTBR14: 129 case R_AARCH64_PLT32: 130 return R_PLT_PC; 131 case R_AARCH64_PREL16: 132 case R_AARCH64_PREL32: 133 case R_AARCH64_PREL64: 134 case R_AARCH64_ADR_PREL_LO21: 135 case R_AARCH64_LD_PREL_LO19: 136 case R_AARCH64_MOVW_PREL_G0: 137 case R_AARCH64_MOVW_PREL_G0_NC: 138 case R_AARCH64_MOVW_PREL_G1: 139 case R_AARCH64_MOVW_PREL_G1_NC: 140 case R_AARCH64_MOVW_PREL_G2: 141 case R_AARCH64_MOVW_PREL_G2_NC: 142 case R_AARCH64_MOVW_PREL_G3: 143 return R_PC; 144 case R_AARCH64_ADR_PREL_PG_HI21: 145 case R_AARCH64_ADR_PREL_PG_HI21_NC: 146 return R_AARCH64_PAGE_PC; 147 case R_AARCH64_LD64_GOT_LO12_NC: 148 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 149 return R_GOT; 150 case R_AARCH64_LD64_GOTPAGE_LO15: 151 return R_AARCH64_GOT_PAGE; 152 case R_AARCH64_ADR_GOT_PAGE: 153 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 154 return R_AARCH64_GOT_PAGE_PC; 155 case R_AARCH64_NONE: 156 return R_NONE; 157 default: 158 error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) + 159 ") against symbol " + toString(s)); 160 return R_NONE; 161 } 162 } 163 164 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const { 165 if (expr == R_RELAX_TLS_GD_TO_IE) { 166 if (type == R_AARCH64_TLSDESC_ADR_PAGE21) 167 return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC; 168 return R_RELAX_TLS_GD_TO_IE_ABS; 169 } 170 return expr; 171 } 172 173 bool AArch64::usesOnlyLowPageBits(RelType type) const { 174 switch (type) { 175 default: 176 return false; 177 case R_AARCH64_ADD_ABS_LO12_NC: 178 case R_AARCH64_LD64_GOT_LO12_NC: 179 case R_AARCH64_LDST128_ABS_LO12_NC: 180 case R_AARCH64_LDST16_ABS_LO12_NC: 181 case R_AARCH64_LDST32_ABS_LO12_NC: 182 case R_AARCH64_LDST64_ABS_LO12_NC: 183 case R_AARCH64_LDST8_ABS_LO12_NC: 184 case R_AARCH64_TLSDESC_ADD_LO12: 185 case R_AARCH64_TLSDESC_LD64_LO12: 186 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 187 return true; 188 } 189 } 190 191 RelType AArch64::getDynRel(RelType type) const { 192 if (type == R_AARCH64_ABS64) 193 return type; 194 return R_AARCH64_NONE; 195 } 196 197 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const { 198 write64le(buf, in.plt->getVA()); 199 } 200 201 void AArch64::writePltHeader(uint8_t *buf) const { 202 const uint8_t pltData[] = { 203 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 204 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 205 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 206 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 207 0x20, 0x02, 0x1f, 0xd6, // br x17 208 0x1f, 0x20, 0x03, 0xd5, // nop 209 0x1f, 0x20, 0x03, 0xd5, // nop 210 0x1f, 0x20, 0x03, 0xd5 // nop 211 }; 212 memcpy(buf, pltData, sizeof(pltData)); 213 214 uint64_t got = in.gotPlt->getVA(); 215 uint64_t plt = in.plt->getVA(); 216 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 217 getAArch64Page(got + 16) - getAArch64Page(plt + 4)); 218 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 219 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 220 } 221 222 void AArch64::writePlt(uint8_t *buf, const Symbol &sym, 223 uint64_t pltEntryAddr) const { 224 const uint8_t inst[] = { 225 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 226 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 227 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[n])) 228 0x20, 0x02, 0x1f, 0xd6 // br x17 229 }; 230 memcpy(buf, inst, sizeof(inst)); 231 232 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 233 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21, 234 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 235 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 236 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 237 } 238 239 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file, 240 uint64_t branchAddr, const Symbol &s, 241 int64_t a) const { 242 // If s is an undefined weak symbol and does not have a PLT entry then it 243 // will be resolved as a branch to the next instruction. 244 if (s.isUndefWeak() && !s.isInPlt()) 245 return false; 246 // ELF for the ARM 64-bit architecture, section Call and Jump relocations 247 // only permits range extension thunks for R_AARCH64_CALL26 and 248 // R_AARCH64_JUMP26 relocation types. 249 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && 250 type != R_AARCH64_PLT32) 251 return false; 252 uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a); 253 return !inBranchRange(type, branchAddr, dst); 254 } 255 256 uint32_t AArch64::getThunkSectionSpacing() const { 257 // See comment in Arch/ARM.cpp for a more detailed explanation of 258 // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to 259 // Thunk have a range of +/- 128 MiB 260 return (128 * 1024 * 1024) - 0x30000; 261 } 262 263 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const { 264 if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 && 265 type != R_AARCH64_PLT32) 266 return true; 267 // The AArch64 call and unconditional branch instructions have a range of 268 // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB. 269 uint64_t range = 270 type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024); 271 if (dst > src) { 272 // Immediate of branch is signed. 273 range -= 4; 274 return dst - src <= range; 275 } 276 return src - dst <= range; 277 } 278 279 static void write32AArch64Addr(uint8_t *l, uint64_t imm) { 280 uint32_t immLo = (imm & 0x3) << 29; 281 uint32_t immHi = (imm & 0x1FFFFC) << 3; 282 uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3); 283 write32le(l, (read32le(l) & ~mask) | immLo | immHi); 284 } 285 286 // Return the bits [Start, End] from Val shifted Start bits. 287 // For instance, getBits(0xF0, 4, 8) returns 0xF. 288 static uint64_t getBits(uint64_t val, int start, int end) { 289 uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1; 290 return (val >> start) & mask; 291 } 292 293 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); } 294 295 // Update the immediate field in a AARCH64 ldr, str, and add instruction. 296 static void or32AArch64Imm(uint8_t *l, uint64_t imm) { 297 or32le(l, (imm & 0xFFF) << 10); 298 } 299 300 // Update the immediate field in an AArch64 movk, movn or movz instruction 301 // for a signed relocation, and update the opcode of a movn or movz instruction 302 // to match the sign of the operand. 303 static void writeSMovWImm(uint8_t *loc, uint32_t imm) { 304 uint32_t inst = read32le(loc); 305 // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk. 306 if (!(inst & (1 << 29))) { 307 // movn or movz. 308 if (imm & 0x10000) { 309 // Change opcode to movn, which takes an inverted operand. 310 imm ^= 0xFFFF; 311 inst &= ~(1 << 30); 312 } else { 313 // Change opcode to movz. 314 inst |= 1 << 30; 315 } 316 } 317 write32le(loc, inst | ((imm & 0xFFFF) << 5)); 318 } 319 320 void AArch64::relocate(uint8_t *loc, const Relocation &rel, 321 uint64_t val) const { 322 switch (rel.type) { 323 case R_AARCH64_ABS16: 324 case R_AARCH64_PREL16: 325 checkIntUInt(loc, val, 16, rel); 326 write16le(loc, val); 327 break; 328 case R_AARCH64_ABS32: 329 case R_AARCH64_PREL32: 330 checkIntUInt(loc, val, 32, rel); 331 write32le(loc, val); 332 break; 333 case R_AARCH64_PLT32: 334 checkInt(loc, val, 32, rel); 335 write32le(loc, val); 336 break; 337 case R_AARCH64_ABS64: 338 case R_AARCH64_PREL64: 339 write64le(loc, val); 340 break; 341 case R_AARCH64_ADD_ABS_LO12_NC: 342 or32AArch64Imm(loc, val); 343 break; 344 case R_AARCH64_ADR_GOT_PAGE: 345 case R_AARCH64_ADR_PREL_PG_HI21: 346 case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21: 347 case R_AARCH64_TLSDESC_ADR_PAGE21: 348 checkInt(loc, val, 33, rel); 349 LLVM_FALLTHROUGH; 350 case R_AARCH64_ADR_PREL_PG_HI21_NC: 351 write32AArch64Addr(loc, val >> 12); 352 break; 353 case R_AARCH64_ADR_PREL_LO21: 354 checkInt(loc, val, 21, rel); 355 write32AArch64Addr(loc, val); 356 break; 357 case R_AARCH64_JUMP26: 358 // Normally we would just write the bits of the immediate field, however 359 // when patching instructions for the cpu errata fix -fix-cortex-a53-843419 360 // we want to replace a non-branch instruction with a branch immediate 361 // instruction. By writing all the bits of the instruction including the 362 // opcode and the immediate (0 001 | 01 imm26) we can do this 363 // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of 364 // the instruction we want to patch. 365 write32le(loc, 0x14000000); 366 LLVM_FALLTHROUGH; 367 case R_AARCH64_CALL26: 368 checkInt(loc, val, 28, rel); 369 or32le(loc, (val & 0x0FFFFFFC) >> 2); 370 break; 371 case R_AARCH64_CONDBR19: 372 case R_AARCH64_LD_PREL_LO19: 373 checkAlignment(loc, val, 4, rel); 374 checkInt(loc, val, 21, rel); 375 or32le(loc, (val & 0x1FFFFC) << 3); 376 break; 377 case R_AARCH64_LDST8_ABS_LO12_NC: 378 case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC: 379 or32AArch64Imm(loc, getBits(val, 0, 11)); 380 break; 381 case R_AARCH64_LDST16_ABS_LO12_NC: 382 case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC: 383 checkAlignment(loc, val, 2, rel); 384 or32AArch64Imm(loc, getBits(val, 1, 11)); 385 break; 386 case R_AARCH64_LDST32_ABS_LO12_NC: 387 case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC: 388 checkAlignment(loc, val, 4, rel); 389 or32AArch64Imm(loc, getBits(val, 2, 11)); 390 break; 391 case R_AARCH64_LDST64_ABS_LO12_NC: 392 case R_AARCH64_LD64_GOT_LO12_NC: 393 case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC: 394 case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC: 395 case R_AARCH64_TLSDESC_LD64_LO12: 396 checkAlignment(loc, val, 8, rel); 397 or32AArch64Imm(loc, getBits(val, 3, 11)); 398 break; 399 case R_AARCH64_LDST128_ABS_LO12_NC: 400 case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC: 401 checkAlignment(loc, val, 16, rel); 402 or32AArch64Imm(loc, getBits(val, 4, 11)); 403 break; 404 case R_AARCH64_LD64_GOTPAGE_LO15: 405 checkAlignment(loc, val, 8, rel); 406 or32AArch64Imm(loc, getBits(val, 3, 14)); 407 break; 408 case R_AARCH64_MOVW_UABS_G0: 409 checkUInt(loc, val, 16, rel); 410 LLVM_FALLTHROUGH; 411 case R_AARCH64_MOVW_UABS_G0_NC: 412 or32le(loc, (val & 0xFFFF) << 5); 413 break; 414 case R_AARCH64_MOVW_UABS_G1: 415 checkUInt(loc, val, 32, rel); 416 LLVM_FALLTHROUGH; 417 case R_AARCH64_MOVW_UABS_G1_NC: 418 or32le(loc, (val & 0xFFFF0000) >> 11); 419 break; 420 case R_AARCH64_MOVW_UABS_G2: 421 checkUInt(loc, val, 48, rel); 422 LLVM_FALLTHROUGH; 423 case R_AARCH64_MOVW_UABS_G2_NC: 424 or32le(loc, (val & 0xFFFF00000000) >> 27); 425 break; 426 case R_AARCH64_MOVW_UABS_G3: 427 or32le(loc, (val & 0xFFFF000000000000) >> 43); 428 break; 429 case R_AARCH64_MOVW_PREL_G0: 430 case R_AARCH64_MOVW_SABS_G0: 431 case R_AARCH64_TLSLE_MOVW_TPREL_G0: 432 checkInt(loc, val, 17, rel); 433 LLVM_FALLTHROUGH; 434 case R_AARCH64_MOVW_PREL_G0_NC: 435 case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC: 436 writeSMovWImm(loc, val); 437 break; 438 case R_AARCH64_MOVW_PREL_G1: 439 case R_AARCH64_MOVW_SABS_G1: 440 case R_AARCH64_TLSLE_MOVW_TPREL_G1: 441 checkInt(loc, val, 33, rel); 442 LLVM_FALLTHROUGH; 443 case R_AARCH64_MOVW_PREL_G1_NC: 444 case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC: 445 writeSMovWImm(loc, val >> 16); 446 break; 447 case R_AARCH64_MOVW_PREL_G2: 448 case R_AARCH64_MOVW_SABS_G2: 449 case R_AARCH64_TLSLE_MOVW_TPREL_G2: 450 checkInt(loc, val, 49, rel); 451 LLVM_FALLTHROUGH; 452 case R_AARCH64_MOVW_PREL_G2_NC: 453 writeSMovWImm(loc, val >> 32); 454 break; 455 case R_AARCH64_MOVW_PREL_G3: 456 writeSMovWImm(loc, val >> 48); 457 break; 458 case R_AARCH64_TSTBR14: 459 checkInt(loc, val, 16, rel); 460 or32le(loc, (val & 0xFFFC) << 3); 461 break; 462 case R_AARCH64_TLSLE_ADD_TPREL_HI12: 463 checkUInt(loc, val, 24, rel); 464 or32AArch64Imm(loc, val >> 12); 465 break; 466 case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC: 467 case R_AARCH64_TLSDESC_ADD_LO12: 468 or32AArch64Imm(loc, val); 469 break; 470 default: 471 llvm_unreachable("unknown relocation"); 472 } 473 } 474 475 void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, 476 uint64_t val) const { 477 // TLSDESC Global-Dynamic relocation are in the form: 478 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 479 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 480 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 481 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 482 // blr x1 483 // And it can optimized to: 484 // movz x0, #0x0, lsl #16 485 // movk x0, #0x10 486 // nop 487 // nop 488 checkUInt(loc, val, 32, rel); 489 490 switch (rel.type) { 491 case R_AARCH64_TLSDESC_ADD_LO12: 492 case R_AARCH64_TLSDESC_CALL: 493 write32le(loc, 0xd503201f); // nop 494 return; 495 case R_AARCH64_TLSDESC_ADR_PAGE21: 496 write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz 497 return; 498 case R_AARCH64_TLSDESC_LD64_LO12: 499 write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk 500 return; 501 default: 502 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 503 } 504 } 505 506 void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, 507 uint64_t val) const { 508 // TLSDESC Global-Dynamic relocation are in the form: 509 // adrp x0, :tlsdesc:v [R_AARCH64_TLSDESC_ADR_PAGE21] 510 // ldr x1, [x0, #:tlsdesc_lo12:v [R_AARCH64_TLSDESC_LD64_LO12] 511 // add x0, x0, :tlsdesc_los:v [R_AARCH64_TLSDESC_ADD_LO12] 512 // .tlsdesccall [R_AARCH64_TLSDESC_CALL] 513 // blr x1 514 // And it can optimized to: 515 // adrp x0, :gottprel:v 516 // ldr x0, [x0, :gottprel_lo12:v] 517 // nop 518 // nop 519 520 switch (rel.type) { 521 case R_AARCH64_TLSDESC_ADD_LO12: 522 case R_AARCH64_TLSDESC_CALL: 523 write32le(loc, 0xd503201f); // nop 524 break; 525 case R_AARCH64_TLSDESC_ADR_PAGE21: 526 write32le(loc, 0x90000000); // adrp 527 relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val); 528 break; 529 case R_AARCH64_TLSDESC_LD64_LO12: 530 write32le(loc, 0xf9400000); // ldr 531 relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val); 532 break; 533 default: 534 llvm_unreachable("unsupported relocation for TLS GD to LE relaxation"); 535 } 536 } 537 538 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, 539 uint64_t val) const { 540 checkUInt(loc, val, 32, rel); 541 542 if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) { 543 // Generate MOVZ. 544 uint32_t regNo = read32le(loc) & 0x1f; 545 write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5)); 546 return; 547 } 548 if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) { 549 // Generate MOVK. 550 uint32_t regNo = read32le(loc) & 0x1f; 551 write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5)); 552 return; 553 } 554 llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); 555 } 556 557 // AArch64 may use security features in variant PLT sequences. These are: 558 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target 559 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used 560 // in the variant Plt sequences are encoded in the Hint space so they can be 561 // deployed on older architectures, which treat the instructions as a nop. 562 // PAC and BTI can be combined leading to the following combinations: 563 // writePltHeader 564 // writePltHeaderBti (no PAC Header needed) 565 // writePlt 566 // writePltBti (BTI only) 567 // writePltPac (PAC only) 568 // writePltBtiPac (BTI and PAC) 569 // 570 // When PAC is enabled the dynamic loader encrypts the address that it places 571 // in the .got.plt using the pacia1716 instruction which encrypts the value in 572 // x17 using the modifier in x16. The static linker places autia1716 before the 573 // indirect branch to x17 to authenticate the address in x17 with the modifier 574 // in x16. This makes it more difficult for an attacker to modify the value in 575 // the .got.plt. 576 // 577 // When BTI is enabled all indirect branches must land on a bti instruction. 578 // The static linker must place a bti instruction at the start of any PLT entry 579 // that may be the target of an indirect branch. As the PLT entries call the 580 // lazy resolver indirectly this must have a bti instruction at start. In 581 // general a bti instruction is not needed for a PLT entry as indirect calls 582 // are resolved to the function address and not the PLT entry for the function. 583 // There are a small number of cases where the PLT address can escape, such as 584 // taking the address of a function or ifunc via a non got-generating 585 // relocation, and a shared library refers to that symbol. 586 // 587 // We use the bti c variant of the instruction which permits indirect branches 588 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI 589 // guarantees that all indirect branches from code requiring BTI protection 590 // will go via x16/x17 591 592 namespace { 593 class AArch64BtiPac final : public AArch64 { 594 public: 595 AArch64BtiPac(); 596 void writePltHeader(uint8_t *buf) const override; 597 void writePlt(uint8_t *buf, const Symbol &sym, 598 uint64_t pltEntryAddr) const override; 599 600 private: 601 bool btiHeader; // bti instruction needed in PLT Header 602 bool btiEntry; // bti instruction needed in PLT Entry 603 bool pacEntry; // autia1716 instruction needed in PLT Entry 604 }; 605 } // namespace 606 607 AArch64BtiPac::AArch64BtiPac() { 608 btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI); 609 // A BTI (Branch Target Indicator) Plt Entry is only required if the 610 // address of the PLT entry can be taken by the program, which permits an 611 // indirect jump to the PLT entry. This can happen when the address 612 // of the PLT entry for a function is canonicalised due to the address of 613 // the function in an executable being taken by a shared library. 614 // FIXME: There is a potential optimization to omit the BTI if we detect 615 // that the address of the PLT entry isn't taken. 616 // The PAC PLT entries require dynamic loader support and this isn't known 617 // from properties in the objects, so we use the command line flag. 618 btiEntry = btiHeader && !config->shared; 619 pacEntry = config->zPacPlt; 620 621 if (btiEntry || pacEntry) { 622 pltEntrySize = 24; 623 ipltEntrySize = 24; 624 } 625 } 626 627 void AArch64BtiPac::writePltHeader(uint8_t *buf) const { 628 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 629 const uint8_t pltData[] = { 630 0xf0, 0x7b, 0xbf, 0xa9, // stp x16, x30, [sp,#-16]! 631 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[2])) 632 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[2]))] 633 0x10, 0x02, 0x00, 0x91, // add x16, x16, Offset(&(.plt.got[2])) 634 0x20, 0x02, 0x1f, 0xd6, // br x17 635 0x1f, 0x20, 0x03, 0xd5, // nop 636 0x1f, 0x20, 0x03, 0xd5 // nop 637 }; 638 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 639 640 uint64_t got = in.gotPlt->getVA(); 641 uint64_t plt = in.plt->getVA(); 642 643 if (btiHeader) { 644 // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C 645 // instruction. 646 memcpy(buf, btiData, sizeof(btiData)); 647 buf += sizeof(btiData); 648 plt += sizeof(btiData); 649 } 650 memcpy(buf, pltData, sizeof(pltData)); 651 652 relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21, 653 getAArch64Page(got + 16) - getAArch64Page(plt + 8)); 654 relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16); 655 relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16); 656 if (!btiHeader) 657 // We didn't add the BTI c instruction so round out size with NOP. 658 memcpy(buf + sizeof(pltData), nopData, sizeof(nopData)); 659 } 660 661 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym, 662 uint64_t pltEntryAddr) const { 663 // The PLT entry is of the form: 664 // [btiData] addrInst (pacBr | stdBr) [nopData] 665 const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c 666 const uint8_t addrInst[] = { 667 0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n])) 668 0x11, 0x02, 0x40, 0xf9, // ldr x17, [x16, Offset(&(.plt.got[n]))] 669 0x10, 0x02, 0x00, 0x91 // add x16, x16, Offset(&(.plt.got[n])) 670 }; 671 const uint8_t pacBr[] = { 672 0x9f, 0x21, 0x03, 0xd5, // autia1716 673 0x20, 0x02, 0x1f, 0xd6 // br x17 674 }; 675 const uint8_t stdBr[] = { 676 0x20, 0x02, 0x1f, 0xd6, // br x17 677 0x1f, 0x20, 0x03, 0xd5 // nop 678 }; 679 const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop 680 681 if (btiEntry) { 682 memcpy(buf, btiData, sizeof(btiData)); 683 buf += sizeof(btiData); 684 pltEntryAddr += sizeof(btiData); 685 } 686 687 uint64_t gotPltEntryAddr = sym.getGotPltVA(); 688 memcpy(buf, addrInst, sizeof(addrInst)); 689 relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21, 690 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr)); 691 relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr); 692 relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr); 693 694 if (pacEntry) 695 memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr)); 696 else 697 memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr)); 698 if (!btiEntry) 699 // We didn't add the BTI c instruction so round out size with NOP. 700 memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData)); 701 } 702 703 static TargetInfo *getTargetInfo() { 704 if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI | 705 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) { 706 static AArch64BtiPac t; 707 return &t; 708 } 709 static AArch64 t; 710 return &t; 711 } 712 713 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); } 714