xref: /freebsd/contrib/llvm-project/lld/ELF/Arch/AArch64.cpp (revision 38a52bd3b5cac3da6f7f6eef3dd050e6aa08ebb3)
1 //===- AArch64.cpp --------------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Symbols.h"
10 #include "SyntheticSections.h"
11 #include "Target.h"
12 #include "Thunks.h"
13 #include "lld/Common/ErrorHandler.h"
14 #include "llvm/Object/ELF.h"
15 #include "llvm/Support/Endian.h"
16 
17 using namespace llvm;
18 using namespace llvm::support::endian;
19 using namespace llvm::ELF;
20 using namespace lld;
21 using namespace lld::elf;
22 
23 // Page(Expr) is the page address of the expression Expr, defined
24 // as (Expr & ~0xFFF). (This applies even if the machine page size
25 // supported by the platform has a different value.)
26 uint64_t elf::getAArch64Page(uint64_t expr) {
27   return expr & ~static_cast<uint64_t>(0xFFF);
28 }
29 
30 namespace {
31 class AArch64 : public TargetInfo {
32 public:
33   AArch64();
34   RelExpr getRelExpr(RelType type, const Symbol &s,
35                      const uint8_t *loc) const override;
36   RelType getDynRel(RelType type) const override;
37   int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override;
38   void writeGotPlt(uint8_t *buf, const Symbol &s) const override;
39   void writePltHeader(uint8_t *buf) const override;
40   void writePlt(uint8_t *buf, const Symbol &sym,
41                 uint64_t pltEntryAddr) const override;
42   bool needsThunk(RelExpr expr, RelType type, const InputFile *file,
43                   uint64_t branchAddr, const Symbol &s,
44                   int64_t a) const override;
45   uint32_t getThunkSectionSpacing() const override;
46   bool inBranchRange(RelType type, uint64_t src, uint64_t dst) const override;
47   bool usesOnlyLowPageBits(RelType type) const override;
48   void relocate(uint8_t *loc, const Relocation &rel,
49                 uint64_t val) const override;
50   RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
51   void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
52                       uint64_t val) const override;
53   void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
54                       uint64_t val) const override;
55   void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
56                       uint64_t val) const override;
57 };
58 } // namespace
59 
60 AArch64::AArch64() {
61   copyRel = R_AARCH64_COPY;
62   relativeRel = R_AARCH64_RELATIVE;
63   iRelativeRel = R_AARCH64_IRELATIVE;
64   gotRel = R_AARCH64_GLOB_DAT;
65   pltRel = R_AARCH64_JUMP_SLOT;
66   symbolicRel = R_AARCH64_ABS64;
67   tlsDescRel = R_AARCH64_TLSDESC;
68   tlsGotRel = R_AARCH64_TLS_TPREL64;
69   pltHeaderSize = 32;
70   pltEntrySize = 16;
71   ipltEntrySize = 16;
72   defaultMaxPageSize = 65536;
73 
74   // Align to the 2 MiB page size (known as a superpage or huge page).
75   // FreeBSD automatically promotes 2 MiB-aligned allocations.
76   defaultImageBase = 0x200000;
77 
78   needsThunks = true;
79 }
80 
81 RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
82                             const uint8_t *loc) const {
83   switch (type) {
84   case R_AARCH64_ABS16:
85   case R_AARCH64_ABS32:
86   case R_AARCH64_ABS64:
87   case R_AARCH64_ADD_ABS_LO12_NC:
88   case R_AARCH64_LDST128_ABS_LO12_NC:
89   case R_AARCH64_LDST16_ABS_LO12_NC:
90   case R_AARCH64_LDST32_ABS_LO12_NC:
91   case R_AARCH64_LDST64_ABS_LO12_NC:
92   case R_AARCH64_LDST8_ABS_LO12_NC:
93   case R_AARCH64_MOVW_SABS_G0:
94   case R_AARCH64_MOVW_SABS_G1:
95   case R_AARCH64_MOVW_SABS_G2:
96   case R_AARCH64_MOVW_UABS_G0:
97   case R_AARCH64_MOVW_UABS_G0_NC:
98   case R_AARCH64_MOVW_UABS_G1:
99   case R_AARCH64_MOVW_UABS_G1_NC:
100   case R_AARCH64_MOVW_UABS_G2:
101   case R_AARCH64_MOVW_UABS_G2_NC:
102   case R_AARCH64_MOVW_UABS_G3:
103     return R_ABS;
104   case R_AARCH64_TLSDESC_ADR_PAGE21:
105     return R_AARCH64_TLSDESC_PAGE;
106   case R_AARCH64_TLSDESC_LD64_LO12:
107   case R_AARCH64_TLSDESC_ADD_LO12:
108     return R_TLSDESC;
109   case R_AARCH64_TLSDESC_CALL:
110     return R_TLSDESC_CALL;
111   case R_AARCH64_TLSLE_ADD_TPREL_HI12:
112   case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
113   case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
114   case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
115   case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
116   case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
117   case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
118   case R_AARCH64_TLSLE_MOVW_TPREL_G0:
119   case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
120   case R_AARCH64_TLSLE_MOVW_TPREL_G1:
121   case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
122   case R_AARCH64_TLSLE_MOVW_TPREL_G2:
123     return R_TPREL;
124   case R_AARCH64_CALL26:
125   case R_AARCH64_CONDBR19:
126   case R_AARCH64_JUMP26:
127   case R_AARCH64_TSTBR14:
128   case R_AARCH64_PLT32:
129     return R_PLT_PC;
130   case R_AARCH64_PREL16:
131   case R_AARCH64_PREL32:
132   case R_AARCH64_PREL64:
133   case R_AARCH64_ADR_PREL_LO21:
134   case R_AARCH64_LD_PREL_LO19:
135   case R_AARCH64_MOVW_PREL_G0:
136   case R_AARCH64_MOVW_PREL_G0_NC:
137   case R_AARCH64_MOVW_PREL_G1:
138   case R_AARCH64_MOVW_PREL_G1_NC:
139   case R_AARCH64_MOVW_PREL_G2:
140   case R_AARCH64_MOVW_PREL_G2_NC:
141   case R_AARCH64_MOVW_PREL_G3:
142     return R_PC;
143   case R_AARCH64_ADR_PREL_PG_HI21:
144   case R_AARCH64_ADR_PREL_PG_HI21_NC:
145     return R_AARCH64_PAGE_PC;
146   case R_AARCH64_LD64_GOT_LO12_NC:
147   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
148     return R_GOT;
149   case R_AARCH64_LD64_GOTPAGE_LO15:
150     return R_AARCH64_GOT_PAGE;
151   case R_AARCH64_ADR_GOT_PAGE:
152   case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
153     return R_AARCH64_GOT_PAGE_PC;
154   case R_AARCH64_NONE:
155     return R_NONE;
156   default:
157     error(getErrorLocation(loc) + "unknown relocation (" + Twine(type) +
158           ") against symbol " + toString(s));
159     return R_NONE;
160   }
161 }
162 
163 RelExpr AArch64::adjustTlsExpr(RelType type, RelExpr expr) const {
164   if (expr == R_RELAX_TLS_GD_TO_IE) {
165     if (type == R_AARCH64_TLSDESC_ADR_PAGE21)
166       return R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC;
167     return R_RELAX_TLS_GD_TO_IE_ABS;
168   }
169   return expr;
170 }
171 
172 bool AArch64::usesOnlyLowPageBits(RelType type) const {
173   switch (type) {
174   default:
175     return false;
176   case R_AARCH64_ADD_ABS_LO12_NC:
177   case R_AARCH64_LD64_GOT_LO12_NC:
178   case R_AARCH64_LDST128_ABS_LO12_NC:
179   case R_AARCH64_LDST16_ABS_LO12_NC:
180   case R_AARCH64_LDST32_ABS_LO12_NC:
181   case R_AARCH64_LDST64_ABS_LO12_NC:
182   case R_AARCH64_LDST8_ABS_LO12_NC:
183   case R_AARCH64_TLSDESC_ADD_LO12:
184   case R_AARCH64_TLSDESC_LD64_LO12:
185   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
186     return true;
187   }
188 }
189 
190 RelType AArch64::getDynRel(RelType type) const {
191   if (type == R_AARCH64_ABS64)
192     return type;
193   return R_AARCH64_NONE;
194 }
195 
196 int64_t AArch64::getImplicitAddend(const uint8_t *buf, RelType type) const {
197   switch (type) {
198   case R_AARCH64_TLSDESC:
199     return read64(buf + 8);
200   case R_AARCH64_NONE:
201     return 0;
202   case R_AARCH64_PREL32:
203     return SignExtend64<32>(read32(buf));
204   case R_AARCH64_ABS64:
205   case R_AARCH64_PREL64:
206     return read64(buf);
207   default:
208     internalLinkerError(getErrorLocation(buf),
209                         "cannot read addend for relocation " + toString(type));
210     return 0;
211   }
212 }
213 
214 void AArch64::writeGotPlt(uint8_t *buf, const Symbol &) const {
215   write64(buf, in.plt->getVA());
216 }
217 
218 void AArch64::writePltHeader(uint8_t *buf) const {
219   const uint8_t pltData[] = {
220       0xf0, 0x7b, 0xbf, 0xa9, // stp    x16, x30, [sp,#-16]!
221       0x10, 0x00, 0x00, 0x90, // adrp   x16, Page(&(.plt.got[2]))
222       0x11, 0x02, 0x40, 0xf9, // ldr    x17, [x16, Offset(&(.plt.got[2]))]
223       0x10, 0x02, 0x00, 0x91, // add    x16, x16, Offset(&(.plt.got[2]))
224       0x20, 0x02, 0x1f, 0xd6, // br     x17
225       0x1f, 0x20, 0x03, 0xd5, // nop
226       0x1f, 0x20, 0x03, 0xd5, // nop
227       0x1f, 0x20, 0x03, 0xd5  // nop
228   };
229   memcpy(buf, pltData, sizeof(pltData));
230 
231   uint64_t got = in.gotPlt->getVA();
232   uint64_t plt = in.plt->getVA();
233   relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
234                 getAArch64Page(got + 16) - getAArch64Page(plt + 4));
235   relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);
236   relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);
237 }
238 
239 void AArch64::writePlt(uint8_t *buf, const Symbol &sym,
240                        uint64_t pltEntryAddr) const {
241   const uint8_t inst[] = {
242       0x10, 0x00, 0x00, 0x90, // adrp x16, Page(&(.plt.got[n]))
243       0x11, 0x02, 0x40, 0xf9, // ldr  x17, [x16, Offset(&(.plt.got[n]))]
244       0x10, 0x02, 0x00, 0x91, // add  x16, x16, Offset(&(.plt.got[n]))
245       0x20, 0x02, 0x1f, 0xd6  // br   x17
246   };
247   memcpy(buf, inst, sizeof(inst));
248 
249   uint64_t gotPltEntryAddr = sym.getGotPltVA();
250   relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,
251                 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));
252   relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);
253   relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);
254 }
255 
256 bool AArch64::needsThunk(RelExpr expr, RelType type, const InputFile *file,
257                          uint64_t branchAddr, const Symbol &s,
258                          int64_t a) const {
259   // If s is an undefined weak symbol and does not have a PLT entry then it will
260   // be resolved as a branch to the next instruction. If it is hidden, its
261   // binding has been converted to local, so we just check isUndefined() here. A
262   // undefined non-weak symbol will have been errored.
263   if (s.isUndefined() && !s.isInPlt())
264     return false;
265   // ELF for the ARM 64-bit architecture, section Call and Jump relocations
266   // only permits range extension thunks for R_AARCH64_CALL26 and
267   // R_AARCH64_JUMP26 relocation types.
268   if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
269       type != R_AARCH64_PLT32)
270     return false;
271   uint64_t dst = expr == R_PLT_PC ? s.getPltVA() : s.getVA(a);
272   return !inBranchRange(type, branchAddr, dst);
273 }
274 
275 uint32_t AArch64::getThunkSectionSpacing() const {
276   // See comment in Arch/ARM.cpp for a more detailed explanation of
277   // getThunkSectionSpacing(). For AArch64 the only branches we are permitted to
278   // Thunk have a range of +/- 128 MiB
279   return (128 * 1024 * 1024) - 0x30000;
280 }
281 
282 bool AArch64::inBranchRange(RelType type, uint64_t src, uint64_t dst) const {
283   if (type != R_AARCH64_CALL26 && type != R_AARCH64_JUMP26 &&
284       type != R_AARCH64_PLT32)
285     return true;
286   // The AArch64 call and unconditional branch instructions have a range of
287   // +/- 128 MiB. The PLT32 relocation supports a range up to +/- 2 GiB.
288   uint64_t range =
289       type == R_AARCH64_PLT32 ? (UINT64_C(1) << 31) : (128 * 1024 * 1024);
290   if (dst > src) {
291     // Immediate of branch is signed.
292     range -= 4;
293     return dst - src <= range;
294   }
295   return src - dst <= range;
296 }
297 
298 static void write32AArch64Addr(uint8_t *l, uint64_t imm) {
299   uint32_t immLo = (imm & 0x3) << 29;
300   uint32_t immHi = (imm & 0x1FFFFC) << 3;
301   uint64_t mask = (0x3 << 29) | (0x1FFFFC << 3);
302   write32le(l, (read32le(l) & ~mask) | immLo | immHi);
303 }
304 
305 // Return the bits [Start, End] from Val shifted Start bits.
306 // For instance, getBits(0xF0, 4, 8) returns 0xF.
307 static uint64_t getBits(uint64_t val, int start, int end) {
308   uint64_t mask = ((uint64_t)1 << (end + 1 - start)) - 1;
309   return (val >> start) & mask;
310 }
311 
312 static void or32le(uint8_t *p, int32_t v) { write32le(p, read32le(p) | v); }
313 
314 // Update the immediate field in a AARCH64 ldr, str, and add instruction.
315 static void or32AArch64Imm(uint8_t *l, uint64_t imm) {
316   or32le(l, (imm & 0xFFF) << 10);
317 }
318 
319 // Update the immediate field in an AArch64 movk, movn or movz instruction
320 // for a signed relocation, and update the opcode of a movn or movz instruction
321 // to match the sign of the operand.
322 static void writeSMovWImm(uint8_t *loc, uint32_t imm) {
323   uint32_t inst = read32le(loc);
324   // Opcode field is bits 30, 29, with 10 = movz, 00 = movn and 11 = movk.
325   if (!(inst & (1 << 29))) {
326     // movn or movz.
327     if (imm & 0x10000) {
328       // Change opcode to movn, which takes an inverted operand.
329       imm ^= 0xFFFF;
330       inst &= ~(1 << 30);
331     } else {
332       // Change opcode to movz.
333       inst |= 1 << 30;
334     }
335   }
336   write32le(loc, inst | ((imm & 0xFFFF) << 5));
337 }
338 
339 void AArch64::relocate(uint8_t *loc, const Relocation &rel,
340                        uint64_t val) const {
341   switch (rel.type) {
342   case R_AARCH64_ABS16:
343   case R_AARCH64_PREL16:
344     checkIntUInt(loc, val, 16, rel);
345     write16(loc, val);
346     break;
347   case R_AARCH64_ABS32:
348   case R_AARCH64_PREL32:
349     checkIntUInt(loc, val, 32, rel);
350     write32(loc, val);
351     break;
352   case R_AARCH64_PLT32:
353     checkInt(loc, val, 32, rel);
354     write32(loc, val);
355     break;
356   case R_AARCH64_ABS64:
357   case R_AARCH64_PREL64:
358     write64(loc, val);
359     break;
360   case R_AARCH64_ADD_ABS_LO12_NC:
361     or32AArch64Imm(loc, val);
362     break;
363   case R_AARCH64_ADR_GOT_PAGE:
364   case R_AARCH64_ADR_PREL_PG_HI21:
365   case R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21:
366   case R_AARCH64_TLSDESC_ADR_PAGE21:
367     checkInt(loc, val, 33, rel);
368     LLVM_FALLTHROUGH;
369   case R_AARCH64_ADR_PREL_PG_HI21_NC:
370     write32AArch64Addr(loc, val >> 12);
371     break;
372   case R_AARCH64_ADR_PREL_LO21:
373     checkInt(loc, val, 21, rel);
374     write32AArch64Addr(loc, val);
375     break;
376   case R_AARCH64_JUMP26:
377     // Normally we would just write the bits of the immediate field, however
378     // when patching instructions for the cpu errata fix -fix-cortex-a53-843419
379     // we want to replace a non-branch instruction with a branch immediate
380     // instruction. By writing all the bits of the instruction including the
381     // opcode and the immediate (0 001 | 01 imm26) we can do this
382     // transformation by placing a R_AARCH64_JUMP26 relocation at the offset of
383     // the instruction we want to patch.
384     write32le(loc, 0x14000000);
385     LLVM_FALLTHROUGH;
386   case R_AARCH64_CALL26:
387     checkInt(loc, val, 28, rel);
388     or32le(loc, (val & 0x0FFFFFFC) >> 2);
389     break;
390   case R_AARCH64_CONDBR19:
391   case R_AARCH64_LD_PREL_LO19:
392     checkAlignment(loc, val, 4, rel);
393     checkInt(loc, val, 21, rel);
394     or32le(loc, (val & 0x1FFFFC) << 3);
395     break;
396   case R_AARCH64_LDST8_ABS_LO12_NC:
397   case R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC:
398     or32AArch64Imm(loc, getBits(val, 0, 11));
399     break;
400   case R_AARCH64_LDST16_ABS_LO12_NC:
401   case R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC:
402     checkAlignment(loc, val, 2, rel);
403     or32AArch64Imm(loc, getBits(val, 1, 11));
404     break;
405   case R_AARCH64_LDST32_ABS_LO12_NC:
406   case R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC:
407     checkAlignment(loc, val, 4, rel);
408     or32AArch64Imm(loc, getBits(val, 2, 11));
409     break;
410   case R_AARCH64_LDST64_ABS_LO12_NC:
411   case R_AARCH64_LD64_GOT_LO12_NC:
412   case R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
413   case R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC:
414   case R_AARCH64_TLSDESC_LD64_LO12:
415     checkAlignment(loc, val, 8, rel);
416     or32AArch64Imm(loc, getBits(val, 3, 11));
417     break;
418   case R_AARCH64_LDST128_ABS_LO12_NC:
419   case R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC:
420     checkAlignment(loc, val, 16, rel);
421     or32AArch64Imm(loc, getBits(val, 4, 11));
422     break;
423   case R_AARCH64_LD64_GOTPAGE_LO15:
424     checkAlignment(loc, val, 8, rel);
425     or32AArch64Imm(loc, getBits(val, 3, 14));
426     break;
427   case R_AARCH64_MOVW_UABS_G0:
428     checkUInt(loc, val, 16, rel);
429     LLVM_FALLTHROUGH;
430   case R_AARCH64_MOVW_UABS_G0_NC:
431     or32le(loc, (val & 0xFFFF) << 5);
432     break;
433   case R_AARCH64_MOVW_UABS_G1:
434     checkUInt(loc, val, 32, rel);
435     LLVM_FALLTHROUGH;
436   case R_AARCH64_MOVW_UABS_G1_NC:
437     or32le(loc, (val & 0xFFFF0000) >> 11);
438     break;
439   case R_AARCH64_MOVW_UABS_G2:
440     checkUInt(loc, val, 48, rel);
441     LLVM_FALLTHROUGH;
442   case R_AARCH64_MOVW_UABS_G2_NC:
443     or32le(loc, (val & 0xFFFF00000000) >> 27);
444     break;
445   case R_AARCH64_MOVW_UABS_G3:
446     or32le(loc, (val & 0xFFFF000000000000) >> 43);
447     break;
448   case R_AARCH64_MOVW_PREL_G0:
449   case R_AARCH64_MOVW_SABS_G0:
450   case R_AARCH64_TLSLE_MOVW_TPREL_G0:
451     checkInt(loc, val, 17, rel);
452     LLVM_FALLTHROUGH;
453   case R_AARCH64_MOVW_PREL_G0_NC:
454   case R_AARCH64_TLSLE_MOVW_TPREL_G0_NC:
455     writeSMovWImm(loc, val);
456     break;
457   case R_AARCH64_MOVW_PREL_G1:
458   case R_AARCH64_MOVW_SABS_G1:
459   case R_AARCH64_TLSLE_MOVW_TPREL_G1:
460     checkInt(loc, val, 33, rel);
461     LLVM_FALLTHROUGH;
462   case R_AARCH64_MOVW_PREL_G1_NC:
463   case R_AARCH64_TLSLE_MOVW_TPREL_G1_NC:
464     writeSMovWImm(loc, val >> 16);
465     break;
466   case R_AARCH64_MOVW_PREL_G2:
467   case R_AARCH64_MOVW_SABS_G2:
468   case R_AARCH64_TLSLE_MOVW_TPREL_G2:
469     checkInt(loc, val, 49, rel);
470     LLVM_FALLTHROUGH;
471   case R_AARCH64_MOVW_PREL_G2_NC:
472     writeSMovWImm(loc, val >> 32);
473     break;
474   case R_AARCH64_MOVW_PREL_G3:
475     writeSMovWImm(loc, val >> 48);
476     break;
477   case R_AARCH64_TSTBR14:
478     checkInt(loc, val, 16, rel);
479     or32le(loc, (val & 0xFFFC) << 3);
480     break;
481   case R_AARCH64_TLSLE_ADD_TPREL_HI12:
482     checkUInt(loc, val, 24, rel);
483     or32AArch64Imm(loc, val >> 12);
484     break;
485   case R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
486   case R_AARCH64_TLSDESC_ADD_LO12:
487     or32AArch64Imm(loc, val);
488     break;
489   case R_AARCH64_TLSDESC:
490     // For R_AARCH64_TLSDESC the addend is stored in the second 64-bit word.
491     write64(loc + 8, val);
492     break;
493   default:
494     llvm_unreachable("unknown relocation");
495   }
496 }
497 
498 void AArch64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel,
499                              uint64_t val) const {
500   // TLSDESC Global-Dynamic relocation are in the form:
501   //   adrp    x0, :tlsdesc:v             [R_AARCH64_TLSDESC_ADR_PAGE21]
502   //   ldr     x1, [x0, #:tlsdesc_lo12:v  [R_AARCH64_TLSDESC_LD64_LO12]
503   //   add     x0, x0, :tlsdesc_los:v     [R_AARCH64_TLSDESC_ADD_LO12]
504   //   .tlsdesccall                       [R_AARCH64_TLSDESC_CALL]
505   //   blr     x1
506   // And it can optimized to:
507   //   movz    x0, #0x0, lsl #16
508   //   movk    x0, #0x10
509   //   nop
510   //   nop
511   checkUInt(loc, val, 32, rel);
512 
513   switch (rel.type) {
514   case R_AARCH64_TLSDESC_ADD_LO12:
515   case R_AARCH64_TLSDESC_CALL:
516     write32le(loc, 0xd503201f); // nop
517     return;
518   case R_AARCH64_TLSDESC_ADR_PAGE21:
519     write32le(loc, 0xd2a00000 | (((val >> 16) & 0xffff) << 5)); // movz
520     return;
521   case R_AARCH64_TLSDESC_LD64_LO12:
522     write32le(loc, 0xf2800000 | ((val & 0xffff) << 5)); // movk
523     return;
524   default:
525     llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
526   }
527 }
528 
529 void AArch64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel,
530                              uint64_t val) const {
531   // TLSDESC Global-Dynamic relocation are in the form:
532   //   adrp    x0, :tlsdesc:v             [R_AARCH64_TLSDESC_ADR_PAGE21]
533   //   ldr     x1, [x0, #:tlsdesc_lo12:v  [R_AARCH64_TLSDESC_LD64_LO12]
534   //   add     x0, x0, :tlsdesc_los:v     [R_AARCH64_TLSDESC_ADD_LO12]
535   //   .tlsdesccall                       [R_AARCH64_TLSDESC_CALL]
536   //   blr     x1
537   // And it can optimized to:
538   //   adrp    x0, :gottprel:v
539   //   ldr     x0, [x0, :gottprel_lo12:v]
540   //   nop
541   //   nop
542 
543   switch (rel.type) {
544   case R_AARCH64_TLSDESC_ADD_LO12:
545   case R_AARCH64_TLSDESC_CALL:
546     write32le(loc, 0xd503201f); // nop
547     break;
548   case R_AARCH64_TLSDESC_ADR_PAGE21:
549     write32le(loc, 0x90000000); // adrp
550     relocateNoSym(loc, R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21, val);
551     break;
552   case R_AARCH64_TLSDESC_LD64_LO12:
553     write32le(loc, 0xf9400000); // ldr
554     relocateNoSym(loc, R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC, val);
555     break;
556   default:
557     llvm_unreachable("unsupported relocation for TLS GD to LE relaxation");
558   }
559 }
560 
561 void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel,
562                              uint64_t val) const {
563   checkUInt(loc, val, 32, rel);
564 
565   if (rel.type == R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21) {
566     // Generate MOVZ.
567     uint32_t regNo = read32le(loc) & 0x1f;
568     write32le(loc, (0xd2a00000 | regNo) | (((val >> 16) & 0xffff) << 5));
569     return;
570   }
571   if (rel.type == R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC) {
572     // Generate MOVK.
573     uint32_t regNo = read32le(loc) & 0x1f;
574     write32le(loc, (0xf2800000 | regNo) | ((val & 0xffff) << 5));
575     return;
576   }
577   llvm_unreachable("invalid relocation for TLS IE to LE relaxation");
578 }
579 
580 AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) {
581   if (!config->relax || config->emachine != EM_AARCH64) {
582     safeToRelaxAdrpLdr = false;
583     return;
584   }
585   // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC
586   // always appear in pairs.
587   size_t i = 0;
588   const size_t size = relocs.size();
589   for (; i != size; ++i) {
590     if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) {
591       if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) {
592         ++i;
593         continue;
594       }
595       break;
596     } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) {
597       break;
598     }
599   }
600   safeToRelaxAdrpLdr = i == size;
601 }
602 
603 bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel,
604                                      const Relocation &addRel, uint64_t secAddr,
605                                      uint8_t *buf) const {
606   // When the address of sym is within the range of ADR then
607   // we may relax
608   // ADRP xn, sym
609   // ADD  xn, xn, :lo12: sym
610   // to
611   // NOP
612   // ADR xn, sym
613   if (!config->relax || adrpRel.type != R_AARCH64_ADR_PREL_PG_HI21 ||
614       addRel.type != R_AARCH64_ADD_ABS_LO12_NC)
615     return false;
616   // Check if the relocations apply to consecutive instructions.
617   if (adrpRel.offset + 4 != addRel.offset)
618     return false;
619   if (adrpRel.sym != addRel.sym)
620     return false;
621   if (adrpRel.addend != 0 || addRel.addend != 0)
622     return false;
623 
624   uint32_t adrpInstr = read32le(buf + adrpRel.offset);
625   uint32_t addInstr = read32le(buf + addRel.offset);
626   // Check if the first instruction is ADRP and the second instruction is ADD.
627   if ((adrpInstr & 0x9f000000) != 0x90000000 ||
628       (addInstr & 0xffc00000) != 0x91000000)
629     return false;
630   uint32_t adrpDestReg = adrpInstr & 0x1f;
631   uint32_t addDestReg = addInstr & 0x1f;
632   uint32_t addSrcReg = (addInstr >> 5) & 0x1f;
633   if (adrpDestReg != addDestReg || adrpDestReg != addSrcReg)
634     return false;
635 
636   Symbol &sym = *adrpRel.sym;
637   // Check if the address difference is within 1MiB range.
638   int64_t val = sym.getVA() - (secAddr + addRel.offset);
639   if (val < -1024 * 1024 || val >= 1024 * 1024)
640     return false;
641 
642   Relocation adrRel = {R_ABS, R_AARCH64_ADR_PREL_LO21, addRel.offset,
643                        /*addend=*/0, &sym};
644   // nop
645   write32le(buf + adrpRel.offset, 0xd503201f);
646   // adr x_<dest_reg>
647   write32le(buf + adrRel.offset, 0x10000000 | adrpDestReg);
648   target->relocate(buf + adrRel.offset, adrRel, val);
649   return true;
650 }
651 
652 bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel,
653                                      const Relocation &ldrRel, uint64_t secAddr,
654                                      uint8_t *buf) const {
655   if (!safeToRelaxAdrpLdr)
656     return false;
657 
658   // When the definition of sym is not preemptible then we may
659   // be able to relax
660   // ADRP xn, :got: sym
661   // LDR xn, [ xn :got_lo12: sym]
662   // to
663   // ADRP xn, sym
664   // ADD xn, xn, :lo_12: sym
665 
666   if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE ||
667       ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC)
668     return false;
669   // Check if the relocations apply to consecutive instructions.
670   if (adrpRel.offset + 4 != ldrRel.offset)
671     return false;
672   // Check if the relocations reference the same symbol and
673   // skip undefined, preemptible and STT_GNU_IFUNC symbols.
674   if (!adrpRel.sym || adrpRel.sym != ldrRel.sym || !adrpRel.sym->isDefined() ||
675       adrpRel.sym->isPreemptible || adrpRel.sym->isGnuIFunc())
676     return false;
677   // Check if the addends of the both relocations are zero.
678   if (adrpRel.addend != 0 || ldrRel.addend != 0)
679     return false;
680   uint32_t adrpInstr = read32le(buf + adrpRel.offset);
681   uint32_t ldrInstr = read32le(buf + ldrRel.offset);
682   // Check if the first instruction is ADRP and the second instruction is LDR.
683   if ((adrpInstr & 0x9f000000) != 0x90000000 ||
684       (ldrInstr & 0x3b000000) != 0x39000000)
685     return false;
686   // Check the value of the sf bit.
687   if (!(ldrInstr >> 31))
688     return false;
689   uint32_t adrpDestReg = adrpInstr & 0x1f;
690   uint32_t ldrDestReg = ldrInstr & 0x1f;
691   uint32_t ldrSrcReg = (ldrInstr >> 5) & 0x1f;
692   // Check if ADPR and LDR use the same register.
693   if (adrpDestReg != ldrDestReg || adrpDestReg != ldrSrcReg)
694     return false;
695 
696   Symbol &sym = *adrpRel.sym;
697   // Check if the address difference is within 4GB range.
698   int64_t val =
699       getAArch64Page(sym.getVA()) - getAArch64Page(secAddr + adrpRel.offset);
700   if (val != llvm::SignExtend64(val, 33))
701     return false;
702 
703   Relocation adrpSymRel = {R_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21,
704                            adrpRel.offset, /*addend=*/0, &sym};
705   Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset,
706                        /*addend=*/0, &sym};
707 
708   // adrp x_<dest_reg>
709   write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg);
710   // add x_<dest reg>, x_<dest reg>
711   write32le(buf + addRel.offset, 0x91000000 | adrpDestReg | (adrpDestReg << 5));
712 
713   target->relocate(buf + adrpSymRel.offset, adrpSymRel,
714                    SignExtend64(getAArch64Page(sym.getVA()) -
715                                     getAArch64Page(secAddr + adrpSymRel.offset),
716                                 64));
717   target->relocate(buf + addRel.offset, addRel, SignExtend64(sym.getVA(), 64));
718   tryRelaxAdrpAdd(adrpSymRel, addRel, secAddr, buf);
719   return true;
720 }
721 
722 // AArch64 may use security features in variant PLT sequences. These are:
723 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target
724 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used
725 // in the variant Plt sequences are encoded in the Hint space so they can be
726 // deployed on older architectures, which treat the instructions as a nop.
727 // PAC and BTI can be combined leading to the following combinations:
728 // writePltHeader
729 // writePltHeaderBti (no PAC Header needed)
730 // writePlt
731 // writePltBti (BTI only)
732 // writePltPac (PAC only)
733 // writePltBtiPac (BTI and PAC)
734 //
735 // When PAC is enabled the dynamic loader encrypts the address that it places
736 // in the .got.plt using the pacia1716 instruction which encrypts the value in
737 // x17 using the modifier in x16. The static linker places autia1716 before the
738 // indirect branch to x17 to authenticate the address in x17 with the modifier
739 // in x16. This makes it more difficult for an attacker to modify the value in
740 // the .got.plt.
741 //
742 // When BTI is enabled all indirect branches must land on a bti instruction.
743 // The static linker must place a bti instruction at the start of any PLT entry
744 // that may be the target of an indirect branch. As the PLT entries call the
745 // lazy resolver indirectly this must have a bti instruction at start. In
746 // general a bti instruction is not needed for a PLT entry as indirect calls
747 // are resolved to the function address and not the PLT entry for the function.
748 // There are a small number of cases where the PLT address can escape, such as
749 // taking the address of a function or ifunc via a non got-generating
750 // relocation, and a shared library refers to that symbol.
751 //
752 // We use the bti c variant of the instruction which permits indirect branches
753 // (br) via x16/x17 and indirect function calls (blr) via any register. The ABI
754 // guarantees that all indirect branches from code requiring BTI protection
755 // will go via x16/x17
756 
757 namespace {
758 class AArch64BtiPac final : public AArch64 {
759 public:
760   AArch64BtiPac();
761   void writePltHeader(uint8_t *buf) const override;
762   void writePlt(uint8_t *buf, const Symbol &sym,
763                 uint64_t pltEntryAddr) const override;
764 
765 private:
766   bool btiHeader; // bti instruction needed in PLT Header and Entry
767   bool pacEntry;  // autia1716 instruction needed in PLT Entry
768 };
769 } // namespace
770 
771 AArch64BtiPac::AArch64BtiPac() {
772   btiHeader = (config->andFeatures & GNU_PROPERTY_AARCH64_FEATURE_1_BTI);
773   // A BTI (Branch Target Indicator) Plt Entry is only required if the
774   // address of the PLT entry can be taken by the program, which permits an
775   // indirect jump to the PLT entry. This can happen when the address
776   // of the PLT entry for a function is canonicalised due to the address of
777   // the function in an executable being taken by a shared library, or
778   // non-preemptible ifunc referenced by non-GOT-generating, non-PLT-generating
779   // relocations.
780   // The PAC PLT entries require dynamic loader support and this isn't known
781   // from properties in the objects, so we use the command line flag.
782   pacEntry = config->zPacPlt;
783 
784   if (btiHeader || pacEntry) {
785     pltEntrySize = 24;
786     ipltEntrySize = 24;
787   }
788 }
789 
790 void AArch64BtiPac::writePltHeader(uint8_t *buf) const {
791   const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
792   const uint8_t pltData[] = {
793       0xf0, 0x7b, 0xbf, 0xa9, // stp    x16, x30, [sp,#-16]!
794       0x10, 0x00, 0x00, 0x90, // adrp   x16, Page(&(.plt.got[2]))
795       0x11, 0x02, 0x40, 0xf9, // ldr    x17, [x16, Offset(&(.plt.got[2]))]
796       0x10, 0x02, 0x00, 0x91, // add    x16, x16, Offset(&(.plt.got[2]))
797       0x20, 0x02, 0x1f, 0xd6, // br     x17
798       0x1f, 0x20, 0x03, 0xd5, // nop
799       0x1f, 0x20, 0x03, 0xd5  // nop
800   };
801   const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
802 
803   uint64_t got = in.gotPlt->getVA();
804   uint64_t plt = in.plt->getVA();
805 
806   if (btiHeader) {
807     // PltHeader is called indirectly by plt[N]. Prefix pltData with a BTI C
808     // instruction.
809     memcpy(buf, btiData, sizeof(btiData));
810     buf += sizeof(btiData);
811     plt += sizeof(btiData);
812   }
813   memcpy(buf, pltData, sizeof(pltData));
814 
815   relocateNoSym(buf + 4, R_AARCH64_ADR_PREL_PG_HI21,
816                 getAArch64Page(got + 16) - getAArch64Page(plt + 8));
817   relocateNoSym(buf + 8, R_AARCH64_LDST64_ABS_LO12_NC, got + 16);
818   relocateNoSym(buf + 12, R_AARCH64_ADD_ABS_LO12_NC, got + 16);
819   if (!btiHeader)
820     // We didn't add the BTI c instruction so round out size with NOP.
821     memcpy(buf + sizeof(pltData), nopData, sizeof(nopData));
822 }
823 
824 void AArch64BtiPac::writePlt(uint8_t *buf, const Symbol &sym,
825                              uint64_t pltEntryAddr) const {
826   // The PLT entry is of the form:
827   // [btiData] addrInst (pacBr | stdBr) [nopData]
828   const uint8_t btiData[] = { 0x5f, 0x24, 0x03, 0xd5 }; // bti c
829   const uint8_t addrInst[] = {
830       0x10, 0x00, 0x00, 0x90,  // adrp x16, Page(&(.plt.got[n]))
831       0x11, 0x02, 0x40, 0xf9,  // ldr  x17, [x16, Offset(&(.plt.got[n]))]
832       0x10, 0x02, 0x00, 0x91   // add  x16, x16, Offset(&(.plt.got[n]))
833   };
834   const uint8_t pacBr[] = {
835       0x9f, 0x21, 0x03, 0xd5,  // autia1716
836       0x20, 0x02, 0x1f, 0xd6   // br   x17
837   };
838   const uint8_t stdBr[] = {
839       0x20, 0x02, 0x1f, 0xd6,  // br   x17
840       0x1f, 0x20, 0x03, 0xd5   // nop
841   };
842   const uint8_t nopData[] = { 0x1f, 0x20, 0x03, 0xd5 }; // nop
843 
844   // needsCopy indicates a non-ifunc canonical PLT entry whose address may
845   // escape to shared objects. isInIplt indicates a non-preemptible ifunc. Its
846   // address may escape if referenced by a direct relocation. The condition is
847   // conservative.
848   bool hasBti = btiHeader && (sym.needsCopy || sym.isInIplt);
849   if (hasBti) {
850     memcpy(buf, btiData, sizeof(btiData));
851     buf += sizeof(btiData);
852     pltEntryAddr += sizeof(btiData);
853   }
854 
855   uint64_t gotPltEntryAddr = sym.getGotPltVA();
856   memcpy(buf, addrInst, sizeof(addrInst));
857   relocateNoSym(buf, R_AARCH64_ADR_PREL_PG_HI21,
858                 getAArch64Page(gotPltEntryAddr) - getAArch64Page(pltEntryAddr));
859   relocateNoSym(buf + 4, R_AARCH64_LDST64_ABS_LO12_NC, gotPltEntryAddr);
860   relocateNoSym(buf + 8, R_AARCH64_ADD_ABS_LO12_NC, gotPltEntryAddr);
861 
862   if (pacEntry)
863     memcpy(buf + sizeof(addrInst), pacBr, sizeof(pacBr));
864   else
865     memcpy(buf + sizeof(addrInst), stdBr, sizeof(stdBr));
866   if (!hasBti)
867     // We didn't add the BTI c instruction so round out size with NOP.
868     memcpy(buf + sizeof(addrInst) + sizeof(stdBr), nopData, sizeof(nopData));
869 }
870 
871 static TargetInfo *getTargetInfo() {
872   if (config->andFeatures & (GNU_PROPERTY_AARCH64_FEATURE_1_BTI |
873                              GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) {
874     static AArch64BtiPac t;
875     return &t;
876   }
877   static AArch64 t;
878   return &t;
879 }
880 
881 TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); }
882